Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
refactor: rework the Docker boundary, image ownership, and other logic (#4484)
Kyle Mendellfd742f3ee594
diff --git a/.env.example b/.env.exampleindex 37c103e34..0d6177d5c 100644--- a/.env.example+++ b/.env.example@@ -42,7 +42,7 @@ JWT_SECRET=your-super-secret-jwt-key-change-this# JWT_REFRESH_EXPIRY=168h# Database Configuration-DATABASE_URL=file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate+DATABASE_URL=file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate# Downgrades are blocked by default; enable only temporarily after taking a backup.ALLOW_DOWNGRADE=false# Optional: GitHub API token used for downgrade migration fetches to avoid rate limits.diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 8e2b67dbc..228208473 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -290,8 +290,11 @@ When `./scripts/development/dev.sh start` fails or the app does not load, valida```bashcurl -f http://localhost:3000curl -f http://localhost:3552/api/health+ curl -f http://localhost:3552/api/health/ready```+ `/api/health` reports process liveness. `/api/health/ready` also pings the database and Docker and returns 503 until both answer; use it for orchestrator readiness probes if you need one.+5. If a service is unhealthy, read the targeted logs first:```bashdiff --git a/backend/api/api.go b/backend/api/api.goindex 74124264e..4722c625a 100644--- a/backend/api/api.go+++ b/backend/api/api.go@@ -61,7 +61,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability""github.com/getarcaneapp/arcane/backend/v2/internal/webhook"- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx")@@ -227,7 +226,7 @@ type HandlerDeps struct {Role *role.ModuleVariable *variable.ModuleUpload *upload.Module- Actors *francis.Runtime+ Health *health.Module}// SetupAPI creates and configures the Huma API attached to the Echo router.@@ -274,7 +273,7 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityAppapi.UseMiddleware(middleware.NewActivityBatchID())registerNormalization(api)- health.RegisterRoutes(api, deps.Actors)+ deps.Health.RegisterRoutes(api)deps.Auth.RegisterRoutes(api)passkey.RegisterPasskeys(api, deps.Passkey, deps.Auth.Service(), deps.User.Service())deps.ApiKey.RegisterRoutes(api)@@ -328,5 +327,5 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityAppfunc SetupAPIForSpec() huma.API {e := echo.New()appCtx := handlerutil.NewActivityAppContext(context.Background()) //nolint:forbidigo // Schema discovery has no running request or application lifecycle.- return SetupAPI(e, e.Group("/api"), appCtx, &config.Config{}, HandlerDeps{})+ return SetupAPI(e, e.Group("/api"), appCtx, &config.Config{}, HandlerDeps{Health: health.New(&health.Service{})})}diff --git a/backend/api/api_test.go b/backend/api/api_test.goindex 194782de8..0e78de956 100644--- a/backend/api/api_test.go+++ b/backend/api/api_test.go@@ -24,6 +24,7 @@ import ("github.com/stretchr/testify/require""github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/health""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil")@@ -342,7 +343,7 @@ func TestVariableMaterializationRoutesAreAgentOnly(t *testing.T) {router.Group("/api"),handlerutil.NewActivityAppContext(t.Context()),&config.Config{AgentMode: true},- HandlerDeps{},+ HandlerDeps{Health: health.New(&health.Service{})},)require.Nil(t, agentAPI.OpenAPI().Paths["/variables"])materialized := agentAPI.OpenAPI().Paths["/environments/{id}/templates/variables"]diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.goindex 3387207ff..d6b360902 100644--- a/backend/api/ws/container_stats.go+++ b/backend/api/ws/container_stats.go@@ -42,10 +42,14 @@ func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {// A reconnect can land exactly on the 5s idle-teardown boundary and load a// hub whose Run has already exited. Drop that hub and retry once so the// client gets a live producer instead of a silent socket.- // The hub owns connection cleanup; preserve request values without request cancellation.+ // The hub owns connection cleanup, so it outlives the request and ends with the app.for attempt := range 2 {- hub := h.getOrCreateContainerStatsHub(containerID)- if wshub.ServeClientWithOnRemove(context.WithoutCancel(c.Request().Context()), hub, conn, onRemove) {+ actual, loaded := h.containerStatsHubs.LoadOrStore(containerID, wshub.NewHub(64))+ hub, _ := actual.(*wshub.Hub)+ if !loaded {+ h.runContainerStatsHub(containerID, hub)+ }+ if wshub.ServeClientWithOnRemove(h.ctx, hub, conn, onRemove) {return nil}h.containerStatsHubs.CompareAndDelete(containerID, hub)@@ -59,34 +63,12 @@ func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {return nil}-func (h *WebSocketHandler) getOrCreateContainerStatsHub(containerID string) *wshub.Hub {- if existing, ok := h.containerStatsHubs.Load(containerID); ok {- if hub, localOk := existing.(*wshub.Hub); localOk {- return hub- }- }-- hub := wshub.NewHub(64)- actual, loaded := h.containerStatsHubs.LoadOrStore(containerID, hub)- if loaded {- if existingHub, ok := actual.(*wshub.Hub); ok {- return existingHub- }- // type assertion failure is impossible in practice, but avoid running- // an unregistered hub if it somehow occurs- return hub- }-- h.runContainerStatsHub(containerID, hub)- return hub-}-// containerStatsErrorDrainGrace is how long a failed stats hub stays alive after// broadcasting its error frame, so clients receive it before the hub shuts down.const containerStatsErrorDrainGrace = 2 * time.Secondfunc (h *WebSocketHandler) runContainerStatsHub(containerID string, hub *wshub.Hub) {- ctx, cancel := context.WithCancel(context.Background()) //nolint:forbidigo // Shared stats hubs own their lifecycle and idle cancellation across requests.+ ctx, cancel := context.WithCancel(h.ctx)var cleanupTimer *time.Timervar cleanupTimerMu sync.Mutexdiff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.goindex 7c1f20e15..8e29180d8 100644--- a/backend/api/ws/handler.go+++ b/backend/api/ws/handler.go@@ -41,6 +41,9 @@ var (// WebSocketHandler consolidates all WebSocket and streaming endpoints.// REST endpoints are handled by Huma handlers.type WebSocketHandler struct {+ // ctx ends after open connections were closed with StatusGoingAway on app shutdown.+ ctx context.Context+ conns sync.MapprojectService *project.ProjectServicecontainerService *container.ContainerServiceswarmService *swarm.SwarmService@@ -80,6 +83,7 @@ type WebSocketHandler struct {}func NewWebSocketHandler(+ appCtx context.Context,group *echo.Group,projectService *project.ProjectService,containerService *container.ContainerService,@@ -89,7 +93,9 @@ func NewWebSocketHandler(authMiddleware *auth.AuthMiddleware,cfg *config.Config,) {+ ctx, cancel := context.WithCancel(context.WithoutCancel(appCtx))handler := &WebSocketHandler{+ ctx: ctx,projectService: projectService,containerService: containerService,swarmService: swarmService,@@ -103,6 +109,18 @@ func NewWebSocketHandler(Build(),checkWSOrigin: httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),}+ // Send the close frames before cancelling hubs, whose teardown closes connections without a handshake.+ context.AfterFunc(appCtx, func() {+ var wg sync.WaitGroup+ handler.conns.Range(func(key, _ any) bool {+ wg.Go(func() {+ _ = key.(*websocket.Conn).Close(websocket.StatusGoingAway, "server shutting down")+ })+ return true+ })+ wg.Wait()+ cancel()+ })observeWebSocketMetricsOnce.Do(func() {if err := defaultWebSocketMetrics.ObserveConnections(otel.Meter(tracing.InstrumentationName)); err != nil {otel.Handle(err)@@ -134,6 +152,7 @@ func (h *WebSocketHandler) acceptWS(c *echo.Context, kind, resourceID string) (*UserAgent: req.Header.Get("User-Agent"),}connID := defaultWebSocketMetrics.RegisterConnection(info)+ h.conns.Store(conn, struct{}{})// The HTTP tracing middleware skips upgrades, so this is a root span unless the client sent a traceparent.parent := otel.GetTextMapPropagator().Extract(req.Context(), propagation.HeaderCarrier(req.Header))_, span := otel.Tracer(tracing.InstrumentationName).Start(parent, "websocket "+kind,@@ -149,6 +168,7 @@ func (h *WebSocketHandler) acceptWS(c *echo.Context, kind, resourceID string) (*attribute.String("user_agent.original", info.UserAgent),))return conn, func() {+ h.conns.Delete(conn)defaultWebSocketMetrics.UnregisterConnection(connID)span.End()}, truediff --git a/backend/api/ws/log_streams.go b/backend/api/ws/log_streams.goindex f40101adb..67db95130 100644--- a/backend/api/ws/log_streams.go+++ b/backend/api/ws/log_streams.go@@ -33,20 +33,6 @@ type wsLogStream struct {seq atomic.Uint64}-func buildLogStreamKey(envID, kind, resourceID, format string, batched, follow bool, tail, since string, timestamps bool) string {- return strings.Join([]string{- envID,- kind,- resourceID,- format,- strconv.FormatBool(batched),- strconv.FormatBool(follow),- tail,- since,- strconv.FormatBool(timestamps),- }, "|")-}-func (h *WebSocketHandler) getOrCreateLogStream(key string, create func(onEmpty func(*wsLogStream)) *wsLogStream) *wsLogStream {h.logStreamsMu.Lock()defer h.logStreamsMu.Unlock()@@ -59,7 +45,7 @@ func (h *WebSocketHandler) getOrCreateLogStream(key string, create func(onEmpty}stream := create(func(stream *wsLogStream) {- h.markLogStreamDone(key, stream)+ h.releaseLogStream(key, stream, true)})stream.key = keystream.refs = 1@@ -67,42 +53,22 @@ func (h *WebSocketHandler) getOrCreateLogStream(key string, create func(onEmptyreturn stream}-func takeLogStreamCancel(stream *wsLogStream) context.CancelFunc {- cancel := stream.cancel- stream.cancel = nil- return cancel-}--func (h *WebSocketHandler) releaseLogStream(key string, stream *wsLogStream) {+// releaseLogStream drops a client reference, or marks the source finished when done is set,+// and tears the stream down once no clients remain.+func (h *WebSocketHandler) releaseLogStream(key string, stream *wsLogStream, done bool) {var cancel context.CancelFunch.logStreamsMu.Lock()- if stream.refs > 0 {- stream.refs--+ if done {+ stream.done = true+ } else {+ stream.refs = max(stream.refs-1, 0)}if stream.refs == 0 {if current, ok := h.logStreams[key]; ok && current == stream {delete(h.logStreams, key)}- cancel = takeLogStreamCancel(stream)- }- h.logStreamsMu.Unlock()-- if cancel != nil {- cancel()- }-}--func (h *WebSocketHandler) markLogStreamDone(key string, stream *wsLogStream) {- var cancel context.CancelFunc-- h.logStreamsMu.Lock()- stream.done = true- if stream.refs == 0 {- if current, ok := h.logStreams[key]; ok && current == stream {- delete(h.logStreams, key)- }- cancel = takeLogStreamCancel(stream)+ cancel, stream.cancel = stream.cancel, nil}h.logStreamsMu.Unlock()@@ -147,16 +113,26 @@ func (h *WebSocketHandler) serveLogStream(return}- streamKey := buildLogStreamKey(c.Param("id"), kind, resourceID, params.format, params.batched, params.follow, params.tail, params.since, params.timestamps)+ streamKey := strings.Join([]string{+ c.Param("id"),+ kind,+ resourceID,+ params.format,+ strconv.FormatBool(params.batched),+ strconv.FormatBool(params.follow),+ params.tail,+ params.since,+ strconv.FormatBool(params.timestamps),+ }, "|")stream := h.getOrCreateLogStream(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {return hubBuilder(streamKey, onEmpty)})release := func() {unregister()- h.releaseLogStream(streamKey, stream)+ h.releaseLogStream(streamKey, stream, false)}- // The hub owns connection cleanup; preserve request values without request cancellation.- if !wshub.ServeClientWithOnRemove(context.WithoutCancel(c.Request().Context()), stream.hub, conn, release) {+ // The hub owns connection cleanup, so it outlives the request and ends with the app.+ if !wshub.ServeClientWithOnRemove(h.ctx, stream.hub, conn, release) {// The stream refcount normally keeps this hub alive, so a stopped hub// here means it was torn down out from under us; drop our reference// rather than leaking the connection and the metrics entry.@@ -226,43 +202,25 @@ func (h *WebSocketHandler) ProjectLogs(c *echo.Context) error {"project",params,streamLogs,- normalizeProjectLogMessage,- normalizeProjectLogText,+ func(line string) wshub.LogMessage {+ level, service, message, timestamp := wshub.NormalizeProjectLine(line)+ return wshub.LogMessage{+ Level: level,+ Message: message,+ Service: service,+ Timestamp: timestamp,+ }+ },+ func(line string) string {+ _, _, message, _ := wshub.NormalizeProjectLine(line)+ return message+ },onEmpty,)})return nil}-func newWSLogStream(key, format string) (*wsLogStream, context.Context) {- ls := &wsLogStream{- hub: wshub.NewHub(1024),- firstSubscriber: make(chan struct{}),- format: format,- key: key,- }- ls.hub.SetOnFirstClient(func() {- close(ls.firstSubscriber)- })-- ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec,forbidigo // Shared log hubs own their lifecycle and idle cancellation across requests.- ls.cancel = cancel-- go ls.hub.Run(ctx)-- return ls, ctx-}--func normalizeProjectLogMessage(line string) wshub.LogMessage {- level, service, message, timestamp := wshub.NormalizeProjectLine(line)- return wshub.LogMessage{- Level: level,- Message: message,- Service: service,- Timestamp: timestamp,- }-}-func normalizeContainerLogMessage(line string) wshub.LogMessage {level, message, timestamp := wshub.NormalizeContainerLine(line)return wshub.LogMessage{@@ -272,11 +230,6 @@ func normalizeContainerLogMessage(line string) wshub.LogMessage {}}-func normalizeProjectLogText(line string) string {- _, _, message, _ := wshub.NormalizeProjectLine(line)- return message-}-func (h *WebSocketHandler) startLogHub(key, resourceID, label string,params logStreamParams,@@ -285,7 +238,18 @@ func (h *WebSocketHandler) startLogHub(normalizeText func(string) string,onEmptyHook func(*wsLogStream),) *wsLogStream {- ls, ctx := newWSLogStream(key, params.format)+ ctx, cancel := context.WithCancel(h.ctx) //nolint:gosec // Cancel is stored on the stream and runs on idle teardown.+ ls := &wsLogStream{+ hub: wshub.NewHub(1024),+ firstSubscriber: make(chan struct{}),+ format: params.format,+ key: key,+ cancel: cancel,+ }+ ls.hub.SetOnFirstClient(func() {+ close(ls.firstSubscriber)+ })+ go ls.hub.Run(ctx)ls.hub.SetOnEmpty(func() {if onEmptyHook != nil {@@ -322,13 +286,13 @@ func (h *WebSocketHandler) startLogSource(return}- h.markLogStreamDone(key, ls)+ h.releaseLogStream(key, ls, true)broadcastLogStreamError(ctx, label, resourceID, params.format, err, ls)return}if ctx.Err() == nil {- h.markLogStreamDone(key, ls)+ h.releaseLogStream(key, ls, true)}}()@@ -346,11 +310,14 @@ func startLogForwarders(if params.format == "json" {messages := mapLogLines(ctx, lines, func(line string) wshub.LogMessage {message := normalizeJSON(line)- message.Seq = ls.seq.Add(1)- if message.Timestamp == "" {- message.Timestamp = wshub.NowRFC3339()+ return wshub.LogMessage{+ Seq: ls.seq.Add(1),+ Level: message.Level,+ Message: message.Message,+ Timestamp: cmp.Or(message.Timestamp, wshub.NowRFC3339()),+ Service: message.Service,+ ContainerID: message.ContainerID,}- return message})if params.batched {diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.goindex 211c68e8e..127e43b0f 100644--- a/backend/api/ws/system_stats.go+++ b/backend/api/ws/system_stats.go@@ -12,7 +12,6 @@ import ("github.com/coder/websocket"systemtypes "github.com/getarcaneapp/arcane/types/v2/system""github.com/labstack/echo/v5"- "github.com/samber/hot""github.com/shirou/gopsutil/v4/cpu""github.com/shirou/gopsutil/v4/disk""github.com/shirou/gopsutil/v4/host"@@ -37,31 +36,6 @@ const (// System WebSocket Endpoints// ============================================================================-// checkRateLimit checks and applies rate limiting for WebSocket connections.-func (h *WebSocketHandler) checkRateLimit(clientIP string) bool {- h.activeConnectionsMu.Lock()- defer h.activeConnectionsMu.Unlock()- if h.activeConnections[clientIP] >= 5 {- return false- }- if h.activeConnections == nil {- h.activeConnections = make(map[string]int)- }- h.activeConnections[clientIP]++- return true-}--// releaseRateLimit decrements the connection counter and cleans up if needed.-func (h *WebSocketHandler) releaseRateLimit(clientIP string) {- h.activeConnectionsMu.Lock()- defer h.activeConnectionsMu.Unlock()- if count := h.activeConnections[clientIP]; count > 1 {- h.activeConnections[clientIP] = count - 1- } else {- delete(h.activeConnections, clientIP)- }-}-func (h *WebSocketHandler) acquireSystemStatsSampler(ctx context.Context, interval time.Duration) bool {h.systemStatsSampler.lifecycleMu.Lock()@@ -77,7 +51,7 @@ func (h *WebSocketHandler) acquireSystemStatsSampler(ctx context.Context, intervreturn waitForSystemStatsSamplerReady(ctx, ready)}- samplerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))+ samplerCtx, cancel := context.WithCancel(h.ctx)ready := make(chan struct{})wake := make(chan struct{}, 1)h.systemStatsSampler.cancel = cancel@@ -206,7 +180,9 @@ func (h *WebSocketHandler) runSystemStatsSampler(ctx context.Context, wake <-chacase <-wake:rearm()case <-ticker.C:- h.updateCPUCache(0)+ if val, ok := h.readCPUUsage(0); ok {+ h.cpuCache.Store(val)+ }h.systemStatsSampler.latest.Store(collect(ctx))rearm()}@@ -217,73 +193,62 @@ func (h *WebSocketHandler) runSystemStatsSampler(ctx context.Context, wake <-chafunc (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.SystemStats {cpuUsage, _ := h.cpuCache.Load()- cpuCount := h.getCPUCount()- memUsed, memTotal := h.getMemoryInfo()- cpuCount, memUsed, memTotal = h.applyCgroupLimits(ctx, cpuCount, memUsed, memTotal)- diskUsed, diskTotal := h.getDiskInfo(ctx)- hostname := h.getHostname()- gpuStats, gpuCount := h.getGPUInfo(ctx)-- return systemtypes.SystemStats{- CPUUsage: cpuUsage,- MemoryUsage: memUsed,- MemoryTotal: memTotal,- DiskUsage: diskUsed,- DiskTotal: diskTotal,- CPUCount: cpuCount,- Architecture: runtime.GOARCH,- Platform: runtime.GOOS,- Hostname: hostname,- GPUCount: gpuCount,- GPUs: gpuStats,- }-}--// getCPUCount returns the number of CPUs.-func (h *WebSocketHandler) getCPUCount() int {- h.initSystemStaticInfo()- return h.systemStaticInfo.cpuCount-}--func (h *WebSocketHandler) initSystemStaticInfo() {h.systemStaticInfo.once.Do(func() {cpuCount, err := cpu.Counts(true)if err != nil {cpuCount = runtime.NumCPU()}-hostInfo, _ := host.Info()- hostname := ""- if hostInfo != nil {- hostname = hostInfo.Hostname- }-h.systemStaticInfo.cpuCount = cpuCount- h.systemStaticInfo.hostname = hostname+ h.systemStaticInfo.hostname = kit.FromPtr(hostInfo).Hostname})-}-// getMemoryInfo returns memory usage and total.-func (h *WebSocketHandler) getMemoryInfo() (uint64, uint64) {- memInfo, _ := mem.VirtualMemory()- if memInfo == nil {- return 0, 0+ var memUsed, memTotal uint64+ if memInfo, _ := mem.VirtualMemory(); memInfo != nil {+ // gopsutil counts ZFS ARC as used memory; treat the reclaimable portion as cache, matching btop/htop.+ memUsed = memInfo.Used - min(memInfo.Used, cgroup.ZFSARCReclaimable())+ memTotal = memInfo.Total+ }+ cpuCount, memUsed, memTotal := h.applyCgroupLimits(ctx, h.systemStaticInfo.cpuCount, memUsed, memTotal)++ diskUsagePath, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {+ path := "/"+ if h.systemService != nil {+ path = h.systemService.GetDiskUsagePath(ctx)+ }+ return map[struct{}]string{{}: path}, nil+ })+ diskUsagePath = kit.Ternary(err != nil || !found, "/", diskUsagePath)+ diskInfo, err := disk.Usage(diskUsagePath)+ if (err != nil || diskInfo == nil || diskInfo.Total == 0) && diskUsagePath != "/" {+ diskInfo, _ = disk.Usage("/")+ }++ var gpuStats []systemtypes.GPUStats+ if h.gpuMonitor != nil && h.gpuMonitor.Enabled() {+ if stats, statsErr := h.gpuMonitor.Stats(ctx); statsErr == nil {+ gpuStats = stats+ }}- // gopsutil counts ZFS ARC as used memory (the kernel excludes it from- // MemAvailable). Treat the reclaimable portion as cache, matching- // btop/htop, so the dashboard does not over-report usage on ZFS hosts.- used := memInfo.Used- if arc := cgroup.ZFSARCReclaimable(); arc > 0 {- used -= min(used, arc)++ return systemtypes.SystemStats{+ CPUUsage: cpuUsage,+ MemoryUsage: memUsed,+ MemoryTotal: memTotal,+ DiskUsage: kit.FromPtr(diskInfo).Used,+ DiskTotal: kit.FromPtr(diskInfo).Total,+ CPUCount: cpuCount,+ Architecture: runtime.GOARCH,+ Platform: runtime.GOOS,+ Hostname: h.systemStaticInfo.hostname,+ GPUCount: len(gpuStats),+ GPUs: gpuStats,}- return used, memInfo.Total}// applyCgroupLimits uses guest resources without applying Docker's// per-container limits to host statistics.-func (h *WebSocketHandler) applyCgroupLimits(- ctx context.Context, cpuCount int, memUsed, memTotal uint64,-) (int, uint64, uint64) {+func (h *WebSocketHandler) applyCgroupLimits(ctx context.Context, cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {if cgroup.IsDockerContainer() {if h.systemService != nil {if used, total, ok := h.systemService.GetDockerHostMemory(ctx); ok {@@ -293,7 +258,7 @@ func (h *WebSocketHandler) applyCgroupLimits(return cpuCount, memUsed, memTotal}cpuCount = applyCPUAffinityLimit(cpuCount)- cgroupLimits := h.getCachedCgroupLimits()+ cgroupLimits := h.cgroupCache.Get()if cgroupLimits == nil {return cpuCount, memUsed, memTotal}@@ -327,39 +292,6 @@ func applyCPUAffinityLimit(cpuCount int) int {return cpuCount}-// getDiskInfo returns disk usage and total.-func (h *WebSocketHandler) getDiskInfo(ctx context.Context) (uint64, uint64) {- diskUsagePath := h.getDiskUsagePath(ctx)- diskInfo, err := disk.Usage(diskUsagePath)- if err != nil || diskInfo == nil || diskInfo.Total == 0 {- if diskUsagePath != "/" {- diskInfo, _ = disk.Usage("/")- }- }- if diskInfo == nil {- return 0, 0- }- return diskInfo.Used, diskInfo.Total-}--// getHostname returns the system hostname.-func (h *WebSocketHandler) getHostname() string {- h.initSystemStaticInfo()- return h.systemStaticInfo.hostname-}--// getGPUInfo returns GPU statistics if monitoring is enabled.-func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStats, int) {- if h.gpuMonitor == nil || !h.gpuMonitor.Enabled() {- return nil, 0- }- gpuData, err := h.gpuMonitor.Stats(ctx)- if err != nil {- return nil, 0- }- return gpuData, len(gpuData)-}-// initializeCPUCache permits unavailable CPU samples but stops on cancellation.func (h *WebSocketHandler) initializeCPUCache(ctx context.Context) bool {result := make(chan float64, 1)@@ -385,35 +317,16 @@ func (h *WebSocketHandler) initializeCPUCache(ctx context.Context) bool {}}-func (h *WebSocketHandler) updateCPUCache(interval time.Duration) {- if val, ok := h.readCPUUsage(interval); ok {- h.cpuCache.Store(val)- }-}-func (h *WebSocketHandler) readCPUUsage(interval time.Duration) (float64, bool) {if h.cpuUsageReader != nil {return h.cpuUsageReader(interval)}-- return defaultReadCPUUsage(interval)-}--func defaultReadCPUUsage(interval time.Duration) (float64, bool) {if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {return vals[0], true}-return 0, false}-func (h *WebSocketHandler) getCachedCgroupLimits() *cgroup.Limits {- if h.cgroupCache == nil {- return nil- }- return h.cgroupCache.Get()-}-// SystemStats streams system stats over WebSocket.//// @Summary Get system stats via WebSocket@@ -424,10 +337,25 @@ func (h *WebSocketHandler) getCachedCgroupLimits() *cgroup.Limits {func (h *WebSocketHandler) SystemStats(c *echo.Context) error {clientIP := c.RealIP()- if !h.checkRateLimit(clientIP) {+ h.activeConnectionsMu.Lock()+ if h.activeConnections[clientIP] >= 5 {+ h.activeConnectionsMu.Unlock()return c.JSON(http.StatusTooManyRequests, map[string]any{"success": false, "error": "Too many concurrent stats connections from this IP"})}- defer h.releaseRateLimit(clientIP)+ if h.activeConnections == nil {+ h.activeConnections = make(map[string]int)+ }+ h.activeConnections[clientIP]+++ h.activeConnectionsMu.Unlock()+ defer func() {+ h.activeConnectionsMu.Lock()+ defer h.activeConnectionsMu.Unlock()+ if count := h.activeConnections[clientIP]; count > 1 {+ h.activeConnections[clientIP] = count - 1+ } else {+ delete(h.activeConnections, clientIP)+ }+ }()conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindSystemStats, "")if !ok {@@ -459,8 +387,15 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {}defer h.releaseSystemStatsSampler(sampleInterval)- go h.readSystemStatsPump(ctx, cancel, conn)- // The pong is serviced by readSystemStatsPump.+ // Single reader for this connection; it also services the keepalive pong. Do not add others.+ go func() {+ for {+ if _, _, err := conn.Read(ctx); err != nil {+ cancel()+ return+ }+ }+ }()go wshub.KeepAlive(ctx, cancel, conn)send := func() error {@@ -489,30 +424,3 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {}}}--// readSystemStatsPump is the single reader for the SystemStats websocket.-// Do not add additional readers for this connection.-func (h *WebSocketHandler) readSystemStatsPump(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn) {- for {- if _, _, err := conn.Read(ctx); err != nil {- cancel()- return- }- }-}--func (h *WebSocketHandler) getDiskUsagePath(ctx context.Context) string {- if h.diskUsagePathCache == nil {- h.diskUsagePathCache = hot.NewHotCache[struct{}, string](hot.LRU, 1).- WithTTL(5 * time.Minute).- Build()- }- path, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {- path := "/"- if h.systemService != nil {- path = h.systemService.GetDiskUsagePath(ctx)- }- return map[struct{}]string{{}: path}, nil- })- return kit.Ternary(err != nil || !found, "/", path)-}diff --git a/backend/api/ws/system_stats_test.go b/backend/api/ws/system_stats_test.goindex a30e385b5..5190e9351 100644--- a/backend/api/ws/system_stats_test.go+++ b/backend/api/ws/system_stats_test.go@@ -69,6 +69,7 @@ func TestSystemStatsSamplerEffectiveInterval(t *testing.T) {func TestSystemStatsSamplerRearmsOnFasterSubscriber(t *testing.T) {collections := make(chan struct{}, 16)h := &WebSocketHandler{+ ctx: t.Context(),cpuUsageReader: func(time.Duration) (float64, bool) { return 0, true },systemStatsCollector: func(context.Context) systemtypes.SystemStats {select {diff --git a/backend/internal/activity/children/pubsub/service.go b/backend/internal/activity/children/pubsub/service.gonew file mode 100644index 000000000..822eb570f--- /dev/null+++ b/backend/internal/activity/children/pubsub/service.go@@ -0,0 +1,287 @@+// Package pubsub fans activity stream events out to per-environment subscribers.+package pubsub++import (+ "cmp"+ "context"+ "log/slog"+ "slices"+ "strings"+ "sync"+ "time"++ "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/samber/mo"+)++const (+ // terminalPublishRetention bounds how long a terminal publish suppresses+ // stale non-terminal snapshots for the same activity. Stale publishers are+ // goroutines already past their commit, so they publish within moments; the+ // retention only has to outlive that gap while keeping the latch map small.+ terminalPublishRetention = 10 * time.Minute++ // subscriberMessageQueueLimit bounds the per-subscriber backlog of "message"+ // events; the oldest message is dropped (and flagged as missed) on overflow.+ subscriberMessageQueueLimit = 256+)++// Service fans activity stream events out to per-environment subscribers.+type Service struct {+ subscribersMu sync.RWMutex+ subscribers map[int]*subscriber+ nextSubID int++ // terminalPublished records when each activity's terminal snapshot was+ // published, so later non-terminal snapshots from slower goroutines can+ // be dropped (see admit). Entries are pruned on terminal publishes once+ // older than terminalPublishRetention.+ terminalPublishedMu sync.Mutex+ terminalPublished map[string]time.Time+}++// subscriber buffers stream events between publishers and one stream+// consumer. "activity" events are coalesced in place per activity ID (only the+// latest pending state matters to the UI), so bulk operations emitting rapid+// progress updates cannot overflow the subscriber and force full-snapshot+// resends. Other events keep arrival order in a FIFO bounded by+// subscriberMessageQueueLimit with drop-oldest on overflow.+type subscriber struct {+ environmentID string+ ch chan activity.StreamEvent+ done chan struct{}+ wake chan struct{}++ mu sync.Mutex+ missed bool+ queue []*pendingStreamEvent+ pendingActivity map[string]*pendingStreamEvent+ messageCount int+}++type pendingStreamEvent struct {+ event activity.StreamEvent+}++// NewService returns an empty subscriber registry.+func NewService() *Service {+ return &Service{+ subscribers: map[int]*subscriber{},+ terminalPublished: map[string]time.Time{},+ }+}++func isCoalescable(event activity.StreamEvent) bool {+ return event.Type == "activity" && event.ActivityID != ""+}++func (sub *subscriber) enqueue(event activity.StreamEvent) {+ sub.mu.Lock()+ if isCoalescable(event) {+ if pending, ok := sub.pendingActivity[event.ActivityID]; ok {+ pending.event = event+ sub.mu.Unlock()+ return+ }+ } else {+ if sub.messageCount >= subscriberMessageQueueLimit {+ sub.dropOldestMessageLocked()+ }+ sub.messageCount+++ }+ entry := &pendingStreamEvent{event: event}+ sub.queue = append(sub.queue, entry)+ if isCoalescable(event) {+ sub.pendingActivity[event.ActivityID] = entry+ }+ sub.mu.Unlock()++ select {+ case sub.wake <- struct{}{}:+ default:+ }+}++func (sub *subscriber) dropOldestMessageLocked() {+ for i, entry := range sub.queue {+ if isCoalescable(entry.event) {+ continue+ }+ sub.queue = append(sub.queue[:i], sub.queue[i+1:]...)+ sub.messageCount--+ sub.missed = true+ ctx := context.Background() //nolint:forbidigo // Subscriber buffer trimming runs outside any request context.+ slog.WarnContext(ctx, "activity subscriber message buffer full; snapshot will be sent on next heartbeat",+ "environmentId", sub.environmentID)+ return+ }+}++func (sub *subscriber) next() mo.Option[activity.StreamEvent] {+ sub.mu.Lock()+ defer sub.mu.Unlock()++ if len(sub.queue) == 0 {+ return mo.None[activity.StreamEvent]()+ }+ entry := sub.queue[0]+ sub.queue = sub.queue[1:]+ event := entry.event+ if isCoalescable(event) {+ if sub.pendingActivity[event.ActivityID] == entry {+ delete(sub.pendingActivity, event.ActivityID)+ }+ } else {+ sub.messageCount--+ }+ return mo.Some(event)+}++func (sub *subscriber) pump() {+ defer close(sub.ch)+ for {+ event, ok := sub.next().Get()+ if !ok {+ select {+ case <-sub.wake:+ continue+ case <-sub.done:+ return+ }+ }+ select {+ case sub.ch <- event:+ case <-sub.done:+ return+ }+ }+}++// Subscribe returns the event channel for environmentID, a drop-check, and an unsubscribe func.+func (s *Service) Subscribe(environmentID string) (<-chan activity.StreamEvent, func() bool, func()) {+ ch := make(chan activity.StreamEvent, 64)+ sub := &subscriber{+ environmentID: cmp.Or(strings.TrimSpace(environmentID), "0"),+ ch: ch,+ done: make(chan struct{}),+ wake: make(chan struct{}, 1),+ pendingActivity: map[string]*pendingStreamEvent{},+ }+ s.subscribersMu.Lock()+ s.nextSubID+++ id := s.nextSubID+ s.subscribers[id] = sub+ s.subscribersMu.Unlock()+ go sub.pump()++ missedEvents := func() bool {+ s.subscribersMu.RLock()+ localSub, ok := s.subscribers[id]+ s.subscribersMu.RUnlock()+ if !ok {+ return false+ }++ localSub.mu.Lock()+ defer localSub.mu.Unlock()+ if !localSub.missed {+ return false+ }+ localSub.missed = false+ return true+ }++ unsubscribe := func() {+ s.subscribersMu.Lock()+ localSub, ok := s.subscribers[id]+ if ok {+ delete(s.subscribers, id)+ }+ s.subscribersMu.Unlock()+ if ok {+ // The pump goroutine owns ch and closes it on shutdown.+ close(localSub.done)+ }+ }++ return ch, missedEvents, unsubscribe+}++// PublishActivity streams an activity snapshot unless a terminal snapshot for+// the same activity was already published.+func (s *Service) PublishActivity(item activity.Activity) {+ if !s.admit(item) {+ return+ }+ s.publish(item.EnvironmentID, activity.StreamEvent{+ Type: "activity",+ ActivityID: item.ID,+ Activity: &item,+ Timestamp: time.Now(),+ })+}++// PublishMessage delivers one activity message to every subscriber of environmentID.+func (s *Service) PublishMessage(environmentID string, activityType activity.Type, message activity.Message) {+ s.publish(environmentID, activity.StreamEvent{+ Type: "message",+ ActivityID: message.ActivityID,+ ActivityType: activityType,+ Message: &message,+ Timestamp: time.Now(),+ })+}++// Reopen clears the terminal latch for an activity whose lifecycle can+// legitimately restart, such as a retried job summary.+func (s *Service) Reopen(activityID string) {+ s.terminalPublishedMu.Lock()+ delete(s.terminalPublished, activityID)+ s.terminalPublishedMu.Unlock()+}++// admit orders activity events across the commit → publish gap. Writes+// publish their snapshot after committing, so a goroutine that committed+// before a terminal write can publish after it; since a terminal activity+// emits no further events to correct the stream (and subscriber coalescing+// would even replace an undelivered terminal event in place), such a stale+// non-terminal snapshot must be dropped, not delivered. Activities never leave+// a terminal status, so once a terminal snapshot is published, any later+// non-terminal snapshot for that ID is stale by construction.+func (s *Service) admit(item activity.Activity) bool {+ s.terminalPublishedMu.Lock()+ defer s.terminalPublishedMu.Unlock()+ if !slices.Contains([]activity.Status{activity.StatusSuccess, activity.StatusFailed, activity.StatusCancelled}, item.Status) {+ _, sealed := s.terminalPublished[item.ID]+ return !sealed+ }+ now := time.Now()+ for id, publishedAt := range s.terminalPublished {+ if now.Sub(publishedAt) > terminalPublishRetention {+ delete(s.terminalPublished, id)+ }+ }+ s.terminalPublished[item.ID] = now+ return true+}++func (s *Service) publish(environmentID string, event activity.StreamEvent) {+ s.subscribersMu.RLock()+ subs := make([]*subscriber, 0, len(s.subscribers))+ for _, sub := range s.subscribers {+ if sub.environmentID == environmentID {+ subs = append(subs, sub)+ }+ }+ s.subscribersMu.RUnlock()++ for _, sub := range subs {+ // Source labeling mutates activity fields independently for each subscriber.+ subscriberEvent := event+ if event.Activity != nil {+ subscriberEvent.Activity = new(*event.Activity)+ }+ subscriberEvent.Activities = slices.Clone(event.Activities)+ sub.enqueue(subscriberEvent)+ }+}diff --git a/backend/internal/activity/children/pubsub/service_test.go b/backend/internal/activity/children/pubsub/service_test.gonew file mode 100644index 000000000..d867f3616--- /dev/null+++ b/backend/internal/activity/children/pubsub/service_test.go@@ -0,0 +1,102 @@+package pubsub++import (+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/stretchr/testify/require"+)++func TestSubscribeMarksMissedEventsWhenBufferFull(t *testing.T) {+ service := NewService()++ events, missedEvents, unsubscribe := service.Subscribe("0")+ defer unsubscribe()++ // Message-type events are not coalesced; overflowing the delivery channel+ // plus the bounded pending FIFO must flag missed events so the stream+ // handler resends a snapshot.+ total := cap(events) + subscriberMessageQueueLimit + 50+ for range total {+ service.publish("0", activity.StreamEvent{Type: "message"})+ }+ require.True(t, missedEvents())+ require.False(t, missedEvents())+}++func TestSubscriberCoalescesProgressEvents(t *testing.T) {+ service := NewService()++ events, missedEvents, unsubscribe := service.Subscribe("0")+ defer unsubscribe()++ const updates = 500+ for i := 1; i <= updates; i++ {+ progress := i * 100 / updates+ service.PublishActivity(activity.Activity{+ ID: "act-1",+ EnvironmentID: "0",+ Status: activity.StatusRunning,+ Progress: &progress,+ })+ }++ received := 0+ deadline := time.After(5 * time.Second)+ for {+ select {+ case event := <-events:+ received+++ require.Equal(t, "activity", event.Type)+ if event.Activity != nil && event.Activity.Progress != nil && *event.Activity.Progress == 100 {+ // The consumer was idle during publishing, so the backlog must+ // have been coalesced instead of delivered event-for-event.+ require.Less(t, received, updates)+ require.False(t, missedEvents())+ return+ }+ case <-deadline:+ require.FailNow(t, "did not receive final coalesced progress event")+ }+ }+}++// A stale non-terminal snapshot published after the activity's terminal event+// — a goroutine that committed before the terminal write but publishes after+// it — is dropped instead of reverting subscribers to a running state no later+// event would correct.+func TestDropsStaleSnapshotAfterTerminalPublish(t *testing.T) {+ service := NewService()++ stale := activity.Activity{+ ID: "act-1",+ EnvironmentID: "0",+ Type: activity.TypeImagePull,+ Status: activity.StatusRunning,+ }+ completed := stale+ completed.Status = activity.StatusSuccess++ events, _, unsubscribe := service.Subscribe("0")+ defer unsubscribe()++ service.PublishActivity(completed)+ select {+ case event := <-events:+ require.Equal(t, "activity", event.Type)+ require.Equal(t, activity.StatusSuccess, event.Activity.Status)+ case <-time.After(5 * time.Second):+ require.FailNow(t, "terminal snapshot was not published")+ }++ require.False(t, service.admit(stale))+ require.True(t, service.admit(completed))++ service.PublishActivity(stale)+ select {+ case localEvent := <-events:+ t.Fatalf("stale non-terminal snapshot reached subscriber: %+v", localEvent)+ case <-time.After(100 * time.Millisecond):+ }+}diff --git a/backend/internal/activity/children/slots/service.go b/backend/internal/activity/children/slots/service.gonew file mode 100644index 000000000..691225cf0--- /dev/null+++ b/backend/internal/activity/children/slots/service.go@@ -0,0 +1,161 @@+// Package slots bounds how many queue-opted activities run concurrently per+// environment and tracks which activity holds each slot.+package slots++import (+ "context"+ "sync"+ "time"++ "github.com/samber/mo"++ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"+)++const (+ maxConcurrentActivitiesSettingKey = "maxConcurrentActivities"+ defaultMaxConcurrentActivities = 5+ // slotWaitRecheckInterval bounds how long a queued waiter can miss a limit+ // increase made while no slots were being released.+ slotWaitRecheckInterval = 2 * time.Second+)++// Service reads the limit from settings on every acquisition attempt, while+// held slots are counted independently of it — so changing the limit+// mid-flight never miscounts running work: existing holders keep occupying+// capacity until they release, and waiters are re-evaluated against the new+// limit.+type Service struct {+ settings *settings.SettingsService++ mu sync.Mutex+ state map[string]*environmentSlots++ // releases maps an activity ID to the release func of the slot it holds;+ // the slot is freed when the activity completes.+ releasesMu sync.Mutex+ releases map[string]func()+}++type environmentSlots struct {+ held int+ // wake is closed (and replaced) on every release to broadcast to waiters;+ // each re-checks against the current limit, so a raised limit can admit+ // more than one of them.+ wake chan struct{}+}++// NewService returns a slot limiter sized from the local settings.+func NewService(localSettings *settings.SettingsService) *Service {+ return &Service{+ settings: localSettings,+ state: map[string]*environmentSlots{},+ releases: map[string]func(){},+ }+}++func (s *Service) stateLocked(environmentID string) *environmentSlots {+ slots := s.state[environmentID]+ if slots == nil {+ slots = &environmentSlots{wake: make(chan struct{})}+ s.state[environmentID] = slots+ }+ return slots+}++// TryAcquire grabs a slot without blocking; None means the caller should+// queue and block via Await. Held slots are counted even while the limit is+// unlimited, so enabling a limit later still sees the work already running.+func (s *Service) TryAcquire(ctx context.Context, environmentID string) mo.Option[func()] {+ if s.settings == nil {+ return mo.Some(func() {})+ }+ limit := s.settings.GetIntSetting(ctx, maxConcurrentActivitiesSettingKey, defaultMaxConcurrentActivities)++ s.mu.Lock()+ defer s.mu.Unlock()+ slots := s.stateLocked(environmentID)+ if limit > 0 && slots.held >= limit {+ return mo.None[func()]()+ }+ slots.held+++ return mo.Some(s.releaser(environmentID))+}++// Await blocks until activityID holds a slot or ctx is cancelled, reporting+// whether a slot was newly acquired. The limit is re-read on every attempt so+// setting changes apply to queued waiters.+func (s *Service) Await(ctx context.Context, activityID, environmentID string) (bool, error) {+ s.releasesMu.Lock()+ _, held := s.releases[activityID]+ s.releasesMu.Unlock()+ if held {+ return false, nil+ }++ for s.settings != nil {+ limit := s.settings.GetIntSetting(ctx, maxConcurrentActivitiesSettingKey, defaultMaxConcurrentActivities)+ s.mu.Lock()+ slots := s.stateLocked(environmentID)+ if limit <= 0 || slots.held < limit {+ slots.held+++ s.mu.Unlock()+ s.Register(activityID, s.releaser(environmentID))+ return true, nil+ }+ wake := slots.wake+ s.mu.Unlock()++ timer := time.NewTimer(slotWaitRecheckInterval)+ select {+ case <-wake:+ case <-timer.C:+ case <-ctx.Done():+ timer.Stop()+ return false, context.Cause(ctx)+ }+ timer.Stop()+ }+ s.Register(activityID, func() {})+ return true, nil+}++// Register records release as the slot held by activityID, freeing any slot+// it previously held.+func (s *Service) Register(activityID string, release func()) {+ s.releasesMu.Lock()+ if existing, ok := s.releases[activityID]; ok {+ existing()+ }+ s.releases[activityID] = release+ s.releasesMu.Unlock()+}++// Release frees the slot held by activityID, if any.+func (s *Service) Release(activityID string) {+ s.releasesMu.Lock()+ release, ok := s.releases[activityID]+ if ok {+ delete(s.releases, activityID)+ }+ s.releasesMu.Unlock()+ if ok {+ release()+ }+}++func (s *Service) releaser(environmentID string) func() {+ var once sync.Once+ return func() {+ once.Do(func() {+ s.mu.Lock()+ slots := s.stateLocked(environmentID)+ if slots.held > 0 {+ slots.held--+ }+ close(slots.wake)+ slots.wake = make(chan struct{})+ s.mu.Unlock()+ })+ }+}diff --git a/backend/internal/activity/children/sweeps/service.go b/backend/internal/activity/children/sweeps/service.gonew file mode 100644index 000000000..873c7f831--- /dev/null+++ b/backend/internal/activity/children/sweeps/service.go@@ -0,0 +1,115 @@+// Package sweeps selects the activity rows that periodic and startup repair+// sweeps finalize.+package sweeps++import (+ "context"+ "slices"+ "time"++ "github.com/getarcaneapp/arcane/types/v2/activity"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+)++const (+ staleImageUpdateCheckAge = 6 * time.Hour+ // abandonedActivityGrace is how old an untracked queued/running activity+ // must be before the periodic sweep fails it. It covers the window between+ // StartActivity's row creation and the worker's Track call.+ abandonedActivityGrace = 2 * time.Minute++ // abandonedActivitySweepLimit bounds one sweep; later sweeps take the rest.+ abandonedActivitySweepLimit = 500+)++var backupActions = []string{+ "create_volume_backup",+ "create_system_backup",+ "run_system_volume_backups",+ "scheduled_volume_backup",+ "scheduled_system_backup",+}++// Candidate is the part of an activity row a sweep decides on.+type Candidate struct {+ ID string+ StartedAt time.Time+ Metadata database.JSON `gorm:"type:text"`+}++// Service runs the candidate queries behind the periodic activity sweeps.+type Service struct {+ db *database.DB+}++// NewService returns a sweeps query service over db.+func NewService(db *database.DB) *Service {+ return &Service{db: db}+}++func (s *Service) activities(ctx context.Context) *gorm.DB {+ return s.db.WithContext(ctx).Table("activities")+}++// StaleImageUpdateChecks returns image update checks left running across a+// prior process lifetime.+func (s *Service) StaleImageUpdateChecks(ctx context.Context) ([]Candidate, error) {+ var rows []Candidate+ err := s.activities(ctx).+ Where("type = ? AND status = ? AND started_at < ?", activity.TypeImageUpdateCheck, activity.StatusRunning, time.Now().Add(-staleImageUpdateCheckAge)).+ Find(&rows).Error+ return rows, err+}++// Abandoned returns active non-job activities past the grace period that are+// not in trackedIDs, oldest first.+func (s *Service) Abandoned(ctx context.Context, trackedIDs []string) ([]Candidate, error) {+ query := s.activities(ctx).+ Select("id", "started_at").+ Where("status IN ? AND started_at < ?", []activity.Status{activity.StatusQueued, activity.StatusRunning}, time.Now().Add(-abandonedActivityGrace)).+ Where("type <> ?", activity.TypeJobRun)+ if len(trackedIDs) > 0 {+ query = query.Where("id NOT IN ?", trackedIDs)+ }+ var rows []Candidate+ err := query.Order("started_at").Limit(abandonedActivitySweepLimit).Find(&rows).Error+ return rows, err+}++// OrphanedQueued returns queued non-job activities outside protectedIDs.+func (s *Service) OrphanedQueued(ctx context.Context, protectedIDs []string) ([]Candidate, error) {+ var rows []Candidate+ err := s.activities(ctx).+ Where("status = ?", activity.StatusQueued).+ Where("type <> ?", activity.TypeJobRun).+ Where("id NOT IN ?", append(protectedIDs, "")).+ Find(&rows).Error+ return rows, err+}++// StaleAutoUpdates returns running auto-update activities outside protectedIDs.+func (s *Service) StaleAutoUpdates(ctx context.Context, protectedIDs []string) ([]Candidate, error) {+ var rows []Candidate+ err := s.activities(ctx).+ Where("type = ? AND status = ?", activity.TypeAutoUpdate, activity.StatusRunning).+ Where("id NOT IN ?", append(protectedIDs, "")).+ Find(&rows).Error+ return rows, err+}++// InterruptedBackups returns active backup activities outside protectedIDs.+func (s *Service) InterruptedBackups(ctx context.Context, protectedIDs []string) ([]Candidate, error) {+ var rows []Candidate+ if err := s.activities(ctx).+ Where("status IN ?", []activity.Status{activity.StatusQueued, activity.StatusRunning}).+ Where("id NOT IN ?", append(protectedIDs, "")).+ Find(&rows).Error; err != nil {+ return nil, err+ }+ return slices.DeleteFunc(rows, func(row Candidate) bool {+ action, _ := row.Metadata["action"].(string)+ return !slices.Contains(backupActions, action)+ }), nil+}diff --git a/backend/internal/activity/handler.go b/backend/internal/activity/handler.goindex 6371f05d3..5704051a2 100644--- a/backend/internal/activity/handler.go+++ b/backend/internal/activity/handler.go@@ -91,16 +91,38 @@ func NewHandler(activityService *ActivityService, localEnvironment EnvironmentDefunc (h *ActivityHandler) ListActivities(ctx context.Context, input *ListActivitiesInput) (*handlerutil.Page[activity.Activity], error) {if input.EnvironmentID != "0" {- return h.proxyListActivitiesInternal(ctx, input)+ unavailable := false+ out, err := h.listRemoteActivities(ctx, input, func(path string) (*base.Paginated[activity.Activity], error) {+ var remote base.Paginated[activity.Activity]+ if proxyErr := h.environment.ProxyJSONRequest(ctx, input.EnvironmentID, http.MethodGet, path, nil, &remote); proxyErr != nil {+ var transportErr *remenv.TransportError+ unavailable = errors.As(proxyErr, &transportErr) && transportErr.Unavailable()+ return nil, handlerutil.TranslateRemoteProxyError(proxyErr)+ }+ return &remote, nil+ })+ if out == nil {+ return nil, err+ }+ if ctxErr := ctx.Err(); ctxErr != nil {+ return nil, ctxErr+ }+ if err != nil && !unavailable {+ return nil, err+ }+ // HTTP responses must not discard stored summaries when the agent is offline.+ // Stream pollers retain the proxy error from the shared listing path.+ out.Body.Success = true+ return out, nil}- params := activityListParamsInternal(input)+ params := activityListParams(input)activities, paginationResp, err := h.activityService.ListActivitiesPaginated(ctx, input.EnvironmentID, params)if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)+ h.applyActivitySourceLabels(ctx, input.EnvironmentID, activities)return &handlerutil.Page[activity.Activity]{Body: base.Paginated[activity.Activity]{@@ -120,16 +142,22 @@ func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInpif err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {if input.EnvironmentID != "0" {- return h.proxyGetActivityInternal(ctx, input)+ path := fmt.Sprintf("/api/environments/0/activities/%s?limit=%d", url.PathEscape(input.ActivityID), input.Limit)+ out, proxyErr := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Detail]](ctx, input.EnvironmentID, http.MethodGet, path, nil)+ if proxyErr != nil {+ return nil, proxyErr+ }+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, &out.Data.Activity)+ return &handlerutil.Out[activity.Detail]{Body: *out}, nil}return nil, huma.Error404NotFound("activity not found")}return nil, huma.Error500InternalServerError(err.Error())}- if !canReadJobActivityInternal(ctx, detail.Activity) {+ if !canReadJobActivity(ctx, detail.Activity) {return nil, huma.Error404NotFound("activity not found")}- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &detail.Activity)+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, &detail.Activity)return &handlerutil.Out[activity.Detail]{Body: base.ApiResponse[activity.Detail]{@@ -146,12 +174,12 @@ func (h *ActivityHandler) ClearHistory(ctx context.Context, input *ClearActivity}if input.EnvironmentID != "0" {- out, proxyClearHistoryErr := h.proxyClearHistoryInternal(ctx, input)- if proxyClearHistoryErr != nil {- return nil, proxyClearHistoryErr+ out, proxyErr := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.ClearHistoryResult]](ctx, input.EnvironmentID, http.MethodDelete, "/api/environments/0/activities/history", nil)+ if proxyErr != nil {+ return nil, proxyErr}- out.Body.Data.Deleted += deleted- return out, nil+ out.Data.Deleted += deleted+ return &handlerutil.Out[activity.ClearHistoryResult]{Body: *out}, nil}return &handlerutil.Out[activity.ClearHistoryResult]{@@ -167,13 +195,26 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActivreturn nil, huma.Error400BadRequest("activity id is required")}- requestedBy := h.cancelRequestedByInternal(ctx, input.RequestedBy)+ // Prefer the authenticated user; a proxying manager forwards its user's name.+ requestedBy := strings.TrimSpace(input.RequestedBy)+ if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {+ requestedBy = cmp.Or(strings.TrimSpace(kit.FromPtr(user.DisplayName)), strings.TrimSpace(user.Username), requestedBy)+ }cancelled, err := h.activityService.CancelActivity(ctx, input.EnvironmentID, input.ActivityID, requestedBy)if err != nil {switch {case errors.Is(err, gorm.ErrRecordNotFound):if input.EnvironmentID != "0" {- return h.proxyCancelActivityInternal(ctx, input)+ path := fmt.Sprintf("/api/environments/0/activities/%s/cancel", url.PathEscape(input.ActivityID))+ if requestedBy != "" {+ path += "?requestedBy=" + url.QueryEscape(requestedBy)+ }+ out, proxyErr := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Activity]](ctx, input.EnvironmentID, http.MethodPost, path, nil)+ if proxyErr != nil {+ return nil, proxyErr+ }+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, &out.Data)+ return &handlerutil.Out[activity.Activity]{Body: *out}, nil}return nil, huma.Error404NotFound("activity not found")case errors.Is(err, ErrActivityNotCancelable):@@ -182,7 +223,7 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActivreturn nil, huma.Error500InternalServerError(err.Error())}}- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, cancelled)+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, cancelled)return &handlerutil.Out[activity.Activity]{Body: base.ApiResponse[activity.Activity]{@@ -192,38 +233,10 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActiv}, nil}-func (h *ActivityHandler) proxyCancelActivityInternal(ctx context.Context, input *CancelActivityInput) (*handlerutil.Out[activity.Activity], error) {- path := fmt.Sprintf("/api/environments/0/activities/%s/cancel", url.PathEscape(input.ActivityID))- if requestedBy := h.cancelRequestedByInternal(ctx, input.RequestedBy); requestedBy != "" {- path += "?requestedBy=" + url.QueryEscape(requestedBy)- }- out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Activity]](ctx, input.EnvironmentID, http.MethodPost, path, nil)- if err != nil {- return nil, err- }- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &out.Data)- return &handlerutil.Out[activity.Activity]{Body: *out}, nil-}--// cancelRequestedByInternal resolves a human-readable name for the cancellation-// audit message, preferring the authenticated user and falling back to a name-// forwarded from a proxying controller.-func (h *ActivityHandler) cancelRequestedByInternal(ctx context.Context, forwarded string) string {- if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {- if user.DisplayName != nil && strings.TrimSpace(*user.DisplayName) != "" {- return strings.TrimSpace(*user.DisplayName)- }- if name := strings.TrimSpace(user.Username); name != "" {- return name- }- }- return strings.TrimSpace(forwarded)-}-func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int, events chan<- activity.StreamEvent) {sendSnapshot := func() bool {activities, _, err := h.activityService.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{- Limit: resolveActivityStreamLimitInternal(limit),+ Limit: resolveActivityStreamLimit(limit),SkipCount: true,})if err != nil {@@ -237,7 +250,7 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,}return false}- h.applyActivitySourceLabelsInternal(ctx, "0", activities)+ h.applyActivitySourceLabels(ctx, "0", activities)return agg.Send(ctx, events, activity.StreamEvent{Type: "snapshot",EnvironmentID: "0",@@ -248,7 +261,7 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,snapshotOK := sendSnapshot()- eventCh, missedEvents, unsubscribe := h.activityService.Subscribe("0")+ eventCh, missedEvents, unsubscribe := h.activityService.pubsub.Subscribe("0")defer unsubscribe()ticker := time.NewTicker(activityStreamHeartbeatInterval)@@ -262,11 +275,16 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,if !ok {return}- if !h.canReadActivityStreamEventInternal(ctx, event) {+ if !h.canReadActivityStreamEvent(ctx, event) {continue}event.EnvironmentID = "0"- h.applyActivityStreamEventSourceLabelInternal(ctx, "0", &event)+ if event.Activity != nil {+ h.applyActivitySourceLabel(ctx, "0", event.Activity)+ }+ if len(event.Activities) > 0 {+ h.applyActivitySourceLabels(ctx, "0", event.Activities)+ }if !agg.Send(ctx, events, event) {return}@@ -299,17 +317,17 @@ func (h *ActivityHandler) RunRemoteStreamPollers(ctx context.Context, ps *authz.func(environment environment.Environment) string {return environment.ID},- activityStreamEnvironmentVersionInternal,+ activityStreamEnvironmentVersion,activityStreamEnvReconcileInterval,"activity stream",func(pollCtx context.Context, environment environment.Environment) {// One shared poller per environment (and snapshot limit) serves// every connected client; this subscriber only forwards its// events onto this client's stream.- key := activityStreamEnvironmentVersionInternal(environment) + ":" + strconv.Itoa(resolveActivityStreamLimitInternal(limit))+ key := activityStreamEnvironmentVersion(environment) + ":" + strconv.Itoa(resolveActivityStreamLimit(limit))h.remoteStreamHub.Subscribe(pollCtx, key,func(runCtx context.Context, publish func(activity.StreamEvent)) {- h.runRemoteActivityStreamPollerInternal(runCtx, environment, limit, publish)+ h.runRemoteActivityStreamPoller(runCtx, environment, limit, publish)},func(event activity.StreamEvent) bool {return agg.Send(pollCtx, events, event)@@ -317,14 +335,14 @@ func (h *ActivityHandler) RunRemoteStreamPollers(ctx context.Context, ps *authz.})}-func activityStreamEnvironmentVersionInternal(localEnvironment environment.Environment) string {+func activityStreamEnvironmentVersion(localEnvironment environment.Environment) string {if localEnvironment.UpdatedAt == nil {return localEnvironment.ID}return localEnvironment.ID + ":" + localEnvironment.UpdatedAt.UTC().Format(time.RFC3339Nano)}-func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, localEnvironment environment.Environment, limit int, publish func(activity.StreamEvent)) {+func (h *ActivityHandler) runRemoteActivityStreamPoller(ctx context.Context, localEnvironment environment.Environment, limit int, publish func(activity.StreamEvent)) {environmentID := localEnvironment.IDlastError := ""lastFingerprint := ""@@ -339,56 +357,58 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont}}- output, err := h.proxyListActivitiesForEnvironmentInternal(ctx, currentEnvironment, &ListActivitiesInput{+ output, err := h.listRemoteActivities(ctx, &ListActivitiesInput{EnvironmentID: environmentID,- Limit: resolveActivityStreamLimitInternal(limit),+ Limit: resolveActivityStreamLimit(limit),Order: "desc",- })- if err != nil {- if ctx.Err() != nil {- return+ }, func(path string) (*base.Paginated[activity.Activity], error) {+ pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)+ defer cancelPoll()+ var out base.Paginated[activity.Activity]+ if err := h.environment.ProxyJSONRequestForEnvironment(pollCtx, currentEnvironment, http.MethodGet, path, nil, &out); err != nil {+ return nil, handlerutil.TranslateRemoteProxyError(err)}- if output != nil {- fingerprint := strconv.FormatUint(kit.Fingerprint(output.Body.Data), 16)- if fingerprint != lastFingerprint {- publish(activity.StreamEvent{- Type: "snapshot",- EnvironmentID: environmentID,- Activities: output.Body.Data,- Timestamp: time.Now(),- })- lastFingerprint = fingerprint- lastError = ""- }+ return &out, nil+ })+ if err != nil && ctx.Err() != nil {+ return+ }+ if err == nil && lastError != "" {+ // Resend the snapshot once the environment recovers.+ lastFingerprint = ""+ }+ if output != nil {+ sourceID := cmp.Or(currentEnvironment.ID, environment.LocalEnvironmentID)+ sourceName := environment.DisplayName(sourceID, currentEnvironment.Name)+ for i := range output.Body.Data {+ applyActivitySource(&output.Body.Data[i], sourceID, sourceName)}- // A failing environment must not end the stream; surface the error- // once per distinct message and keep polling.- if msg := err.Error(); msg != lastError {- lastError = msg+ if fingerprint := strconv.FormatUint(kit.Fingerprint(output.Body.Data), 16); fingerprint != lastFingerprint {publish(activity.StreamEvent{- Type: "error",+ Type: "snapshot",EnvironmentID: environmentID,- Error: msg,+ Activities: output.Body.Data,Timestamp: time.Now(),})+ lastFingerprint = fingerprint+ lastError = ""}- return}- if lastError != "" {- lastFingerprint = ""- }- lastError = ""- fingerprint := strconv.FormatUint(kit.Fingerprint(output.Body.Data), 16)- if fingerprint == lastFingerprint {+ if err == nil {+ lastError = ""return}- publish(activity.StreamEvent{- Type: "snapshot",- EnvironmentID: environmentID,- Activities: output.Body.Data,- Timestamp: time.Now(),- })- lastFingerprint = fingerprint+ // A failing environment must not end the stream; surface the error+ // once per distinct message and keep polling.+ if msg := err.Error(); msg != lastError {+ lastError = msg+ publish(activity.StreamEvent{+ Type: "error",+ EnvironmentID: environmentID,+ Error: msg,+ Timestamp: time.Now(),+ })+ }}poll()@@ -406,73 +426,24 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont}}-func (h *ActivityHandler) proxyGetActivityInternal(ctx context.Context, input *GetActivityInput) (*handlerutil.Out[activity.Detail], error) {- path := fmt.Sprintf("/api/environments/0/activities/%s?limit=%d", url.PathEscape(input.ActivityID), input.Limit)- out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Detail]](ctx, input.EnvironmentID, http.MethodGet, path, nil)- if err != nil {- return nil, err- }- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &out.Data.Activity)- return &handlerutil.Out[activity.Detail]{Body: *out}, nil-}--func (h *ActivityHandler) proxyClearHistoryInternal(ctx context.Context, input *ClearActivityHistoryInput) (*handlerutil.Out[activity.ClearHistoryResult], error) {- out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.ClearHistoryResult]](ctx, input.EnvironmentID, http.MethodDelete, "/api/environments/0/activities/history", nil)- if err != nil {- return nil, err- }- return &handlerutil.Out[activity.ClearHistoryResult]{Body: *out}, nil-}--func (h *ActivityHandler) applyActivitySourceLabelsInternal(ctx context.Context, environmentID string, activities []activity.Activity) {- sourceID, sourceName := h.resolveActivitySourceInternal(ctx, environmentID)- for i := range activities {- applyActivitySourceInternal(&activities[i], sourceID, sourceName)- }-}--func (h *ActivityHandler) applyActivitySourceLabelInternal(ctx context.Context, environmentID string, item *activity.Activity) {- sourceID, sourceName := h.resolveActivitySourceInternal(ctx, environmentID)- applyActivitySourceInternal(item, sourceID, sourceName)-}--func (h *ActivityHandler) applyActivityStreamEventSourceLabelInternal(ctx context.Context, environmentID string, event *activity.StreamEvent) {- if event == nil || (event.Activity == nil && len(event.Activities) == 0) {- return- }- sourceID, sourceName := h.resolveActivitySourceInternal(ctx, environmentID)- if event.Activity != nil {- applyActivitySourceInternal(event.Activity, sourceID, sourceName)- }- for i := range event.Activities {- applyActivitySourceInternal(&event.Activities[i], sourceID, sourceName)+func (h *ActivityHandler) applyActivitySourceLabels(ctx context.Context, environmentID string, activities []activity.Activity) {+ sourceID := cmp.Or(environmentID, environment.LocalEnvironmentID)+ sourceName := environment.DisplayName(sourceID, "")+ if h.environment.ResolveEnvironmentName != nil {+ sourceName = h.environment.ResolveEnvironmentName(ctx, sourceID)}-}--func applyActivitySourceLabelsForEnvironmentInternal(environmentModel environment.Environment, activities []activity.Activity) {- sourceID, sourceName := activitySourceFromEnvironmentInternal(environmentModel)for i := range activities {- applyActivitySourceInternal(&activities[i], sourceID, sourceName)+ applyActivitySource(&activities[i], sourceID, sourceName)}}-func activitySourceFromEnvironmentInternal(environmentModel environment.Environment) (string, string) {- environmentID := cmp.Or(environmentModel.ID, environment.LocalEnvironmentID)- return environmentID, environment.DisplayName(environmentID, environmentModel.Name)-}--func (h *ActivityHandler) resolveActivitySourceInternal(ctx context.Context, environmentID string) (string, string) {- environmentID = cmp.Or(environmentID, environment.LocalEnvironmentID)- if h.environment.ResolveEnvironmentName != nil {- return environmentID, h.environment.ResolveEnvironmentName(ctx, environmentID)- }- return environmentID, environment.DisplayName(environmentID, "")+func (h *ActivityHandler) applyActivitySourceLabel(ctx context.Context, environmentID string, item *activity.Activity) {+ items := []activity.Activity{*item}+ h.applyActivitySourceLabels(ctx, environmentID, items)+ *item = items[0]}-func applyActivitySourceInternal(item *activity.Activity, sourceID, sourceName string) {- if item == nil {- return- }+func applyActivitySource(item *activity.Activity, sourceID, sourceName string) {item.SourceEnvironmentID = sourceIDitem.SourceEnvironmentName = sourceNameitem.EnvironmentID = sourceID@@ -482,120 +453,48 @@ func applyActivitySourceInternal(item *activity.Activity, sourceID, sourceName s}}-func activityListQueryInternal(input *ListActivitiesInput) url.Values {- values := url.Values{}- values.Set("start", strconv.Itoa(input.Start))- values.Set("limit", strconv.Itoa(input.Limit))- if input.Search != "" {- values.Set("search", input.Search)- }- if input.Sort != "" {- values.Set("sort", input.Sort)- }- if input.Order != "" {- values.Set("order", input.Order)- }- if input.Status != "" {- values.Set("status", input.Status)- }- if input.Type != "" {- values.Set("type", input.Type)- }- if input.ResourceType != "" {- values.Set("resourceType", input.ResourceType)- }- return values-}--func resolveActivityStreamLimitInternal(limit int) int {- if limit <= 0 {- return 50- }- return kit.Ternary(limit > 100, 100, limit)+func resolveActivityStreamLimit(limit int) int {+ return min(kit.Ternary(limit <= 0, 50, limit), 100)}-func activityListParamsInternal(input *ListActivitiesInput) pagination.QueryParams {+// activityListParams maps list input onto pagination params and its filters.+func activityListParams(input *ListActivitiesInput) pagination.QueryParams {params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)- params.Filters["status"] = input.Status- params.Filters["type"] = input.Type- params.Filters["resourceType"] = input.ResourceType- return params-}--func (h *ActivityHandler) proxyListActivitiesInternal(ctx context.Context, input *ListActivitiesInput) (*handlerutil.Page[activity.Activity], error) {- unavailable := false- out, err := h.listRemoteActivitiesInternal(ctx, input, func(path string) (*base.Paginated[activity.Activity], error) {- var remote base.Paginated[activity.Activity]- if proxyErr := h.environment.ProxyJSONRequest(ctx, input.EnvironmentID, http.MethodGet, path, nil, &remote); proxyErr != nil {- var transportErr *remenv.TransportError- unavailable = errors.As(proxyErr, &transportErr) && transportErr.Unavailable()- return nil, handlerutil.TranslateRemoteProxyError(proxyErr)- }- return &remote, nil+ maps.Copy(params.Filters, map[string]string{+ "status": input.Status,+ "type": input.Type,+ "resourceType": input.ResourceType,})- if out == nil {- return nil, err- }- if ctxErr := ctx.Err(); ctxErr != nil {- return nil, ctxErr- }- if err != nil && !unavailable {- return nil, err- }- // HTTP responses must not discard stored summaries when the agent is offline.- // Stream pollers retain the proxy error from the shared listing path.- out.Body.Success = true- return out, nil-}--func (- h *ActivityHandler,-) proxyListActivitiesForEnvironmentInternal(- ctx context.Context,- localEnvironment environment.Environment,- input *ListActivitiesInput,-) (- *handlerutil.Page[activity.Activity],- error,-) {- out, err := h.listRemoteActivitiesInternal(ctx, input, func(path string) (*base.Paginated[activity.Activity], error) {- pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)- defer cancelPoll()- var out base.Paginated[activity.Activity]- if err := h.environment.ProxyJSONRequestForEnvironment(pollCtx, localEnvironment, http.MethodGet, path, nil, &out); err != nil {- return nil, handlerutil.TranslateRemoteProxyError(err)- }- return &out, nil- })- if out != nil {- applyActivitySourceLabelsForEnvironmentInternal(localEnvironment, out.Body.Data)- }- return out, err+ return params}-func (- h *ActivityHandler,-) listRemoteActivitiesInternal(+func (h *ActivityHandler) listRemoteActivities(ctx context.Context,input *ListActivitiesInput,- fetch func(- string,- ) (- *base.Paginated[activity.Activity],- error,- ),-) (- *handlerutil.Page[activity.Activity],- error,-) {- params := normalizeRemoteActivityParamsInternal(activityListParamsInternal(input))+ fetch func(string) (*base.Paginated[activity.Activity], error),+) (*handlerutil.Page[activity.Activity], error) {+ params := normalizeRemoteActivityParams(activityListParams(input))// The agent only ships the newest window the merged page can use, not its// whole history; the stream poller repeats this every few seconds.window := kit.Ternary(params.Limit == -1, -1, params.Start+params.Limit)- remoteActivities, remoteTotal, remoteErr := collectActivityWindowInternal(window, func(start, limit int) ([]activity.Activity, int64, error) {- page := *input- page.Start, page.Limit = start, limit- remote, err := fetch("/api/environments/0/activities?" + activityListQueryInternal(&page).Encode())+ remoteActivities, remoteTotal, remoteErr := collectActivityWindow(window, func(start, limit int) ([]activity.Activity, int64, error) {+ query := url.Values{+ "start": {strconv.Itoa(start)},+ "limit": {strconv.Itoa(limit)},+ }+ for key, value := range map[string]string{+ "search": input.Search,+ "sort": input.Sort,+ "order": input.Order,+ "status": input.Status,+ "type": input.Type,+ "resourceType": input.ResourceType,+ } {+ if value != "" {+ query.Set(key, value)+ }+ }+ remote, err := fetch("/api/environments/0/activities?" + query.Encode())if err != nil {return nil, 0, err}@@ -607,7 +506,7 @@ func (if err != nil {return nil, huma.Error500InternalServerError(err.Error())}- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)+ h.applyActivitySourceLabels(ctx, input.EnvironmentID, activities)return &handlerutil.Page[activity.Activity]{Body: base.Paginated[activity.Activity]{Success: remoteErr == nil,Data: activities,diff --git a/backend/internal/activity/helpers.go b/backend/internal/activity/helpers.goindex 1ff5f22fe..387c54530 100644--- a/backend/internal/activity/helpers.go+++ b/backend/internal/activity/helpers.go@@ -3,136 +3,16 @@ package activityimport ("cmp""context"- "sync"- "time""github.com/getarcaneapp/arcane/types/v2/activity"- "github.com/samber/mo""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/middleware"- "github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/authz")-const (- maxConcurrentActivitiesSettingKey = "maxConcurrentActivities"- defaultMaxConcurrentActivities = 5- // slotWaitRecheckInterval bounds how long a queued waiter can miss a limit- // increase made while no slots were being released.- slotWaitRecheckInterval = 2 * time.Second-)--// activitySlotLimiter bounds how many queue-opted activities run concurrently-// per environment. The limit is read from settings on every acquisition-// attempt, while held slots are counted independently of it — so changing the-// limit mid-flight never miscounts running work: existing holders keep-// occupying capacity until they release, and waiters are re-evaluated against-// the new limit.-type activitySlotLimiter struct {- settings *settings.SettingsService-- mu sync.Mutex- state map[string]*activityEnvironmentSlots-}--type activityEnvironmentSlots struct {- held int- // wake is closed (and replaced) on every release to broadcast to waiters;- // each re-checks against the current limit, so a raised limit can admit- // more than one of them.- wake chan struct{}-}--func newActivitySlotLimiterInternal(localSettings *settings.SettingsService) *activitySlotLimiter {- return &activitySlotLimiter{- settings: localSettings,- state: map[string]*activityEnvironmentSlots{},- }-}--func (l *activitySlotLimiter) limitInternal(ctx context.Context) int {- return l.settings.GetIntSetting(ctx, maxConcurrentActivitiesSettingKey, defaultMaxConcurrentActivities)-}--func (l *activitySlotLimiter) stateForLockedInternal(environmentID string) *activityEnvironmentSlots {- slots := l.state[environmentID]- if slots == nil {- slots = &activityEnvironmentSlots{wake: make(chan struct{})}- l.state[environmentID] = slots- }- return slots-}--// tryAcquireInternal grabs a slot without blocking; None means the caller-// should queue and block via acquireInternal. Held slots are counted even-// while the limit is unlimited, so enabling a limit later still sees the work-// already running.-func (l *activitySlotLimiter) tryAcquireInternal(ctx context.Context, environmentID string) mo.Option[func()] {- if l == nil || l.settings == nil {- return mo.Some(func() {})- }- limit := l.limitInternal(ctx)-- l.mu.Lock()- defer l.mu.Unlock()- slots := l.stateForLockedInternal(environmentID)- if limit > 0 && slots.held >= limit {- return mo.None[func()]()- }- slots.held++- return mo.Some(l.releaseOnceInternal(environmentID))-}--// acquireInternal blocks until a slot frees or ctx is cancelled. The limit is-// re-read on every attempt so setting changes apply to queued waiters.-func (l *activitySlotLimiter) acquireInternal(ctx context.Context, environmentID string) (func(), error) {- if l == nil || l.settings == nil {- return func() {}, nil- }-- for {- limit := l.limitInternal(ctx)- l.mu.Lock()- slots := l.stateForLockedInternal(environmentID)- if limit <= 0 || slots.held < limit {- slots.held++- l.mu.Unlock()- return l.releaseOnceInternal(environmentID), nil- }- wake := slots.wake- l.mu.Unlock()-- timer := time.NewTimer(slotWaitRecheckInterval)- select {- case <-wake:- case <-timer.C:- case <-ctx.Done():- timer.Stop()- return nil, context.Cause(ctx)- }- timer.Stop()- }-}--func (l *activitySlotLimiter) releaseOnceInternal(environmentID string) func() {- var once sync.Once- return func() {- once.Do(func() {- l.mu.Lock()- slots := l.stateForLockedInternal(environmentID)- if slots.held > 0 {- slots.held--- }- close(slots.wake)- slots.wake = make(chan struct{})- l.mu.Unlock()- })- }-}-// Job summaries live in their target environment's environment_id column.-func scopeJobActivityVisibilityInternal(ctx context.Context, query *gorm.DB, environmentID, permission string) *gorm.DB {+func scopeJobActivityVisibility(ctx context.Context, query *gorm.DB, environmentID, permission string) *gorm.DB {permissions, present := middleware.PermissionsFromContext(ctx)if !present || permissions.Allows(permission, environmentID) {return query@@ -140,7 +20,7 @@ func scopeJobActivityVisibilityInternal(ctx context.Context, query *gorm.DB, envreturn query.Where("type <> ?", activity.TypeJobRun)}-func canReadJobActivityInternal(ctx context.Context, item activity.Activity) bool {+func canReadJobActivity(ctx context.Context, item activity.Activity) bool {if item.Type != activity.TypeJobRun {return true}@@ -153,9 +33,9 @@ func canReadJobActivityInternal(ctx context.Context, item activity.Activity) booreturn permissions.Allows(authz.PermActivitiesRead, environmentID)}-func (h *ActivityHandler) canReadActivityStreamEventInternal(ctx context.Context, event activity.StreamEvent) bool {+func (h *ActivityHandler) canReadActivityStreamEvent(ctx context.Context, event activity.StreamEvent) bool {if event.Activity != nil {- return canReadJobActivityInternal(ctx, *event.Activity)+ return canReadJobActivity(ctx, *event.Activity)}if event.ActivityID == "" {return true@@ -171,5 +51,5 @@ func (h *ActivityHandler) canReadActivityStreamEventInternal(ctx context.Contextif err := h.activityService.db.WithContext(ctx).Select("type", "environment_id", "metadata").Where("id = ? AND environment_id = ?", event.ActivityID, "0").First(&model).Error; err != nil {return false}- return canReadJobActivityInternal(ctx, activity.Activity{Type: model.Type, EnvironmentID: model.EnvironmentID, Metadata: model.Metadata})+ return canReadJobActivity(ctx, activity.Activity{Type: model.Type, EnvironmentID: model.EnvironmentID, Metadata: model.Metadata})}diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.goindex d229abe88..df1cea574 100644--- a/backend/internal/activity/service.go+++ b/backend/internal/activity/service.go@@ -21,6 +21,9 @@ import ("gorm.io/gorm""gorm.io/gorm/clause"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/pubsub"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/slots"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/sweeps""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/settings"@@ -36,29 +39,8 @@ const (defaultActivityRetentionDays = 30defaultActivityHistoryLimit = 1000defaultActivityMessages = 500- staleImageUpdateCheckAge = 6 * time.Hour- // abandonedActivityGrace is how old an untracked queued/running activity- // must be before the periodic sweep fails it. It covers the window between- // StartActivity's row creation and the worker's Track call.- abandonedActivityGrace = 2 * time.Minute- // abandonedActivitySweepLimit bounds one sweep; later sweeps take the rest.- abandonedActivitySweepLimit = 500-- // terminalPublishRetention bounds how long a terminal publish suppresses- // stale non-terminal snapshots for the same activity. Stale publishers are- // goroutines already past their commit, so they publish within moments; the- // retention only has to outlive that gap while keeping the latch map small.- terminalPublishRetention = 10 * time.Minute-- // ErrActivityNotCancelable indicates the activity has already reached a terminal- // state and can no longer be cancelled.-- // subscriberMessageQueueLimit bounds the per-subscriber backlog of "message"- // events; the oldest message is dropped (and flagged as missed) on overflow.- subscriberMessageQueueLimit = 256-- cancelledMessageInternal = "Cancelled by user"+ cancelledMessage = "Cancelled by user"deleteActivitiesBatchSize = 500)@@ -66,11 +48,10 @@ const (var ErrActivityNotCancelable = errors.New("activity is not cancelable")type ActivityService struct {- db *database.DB-- subscribersMu sync.RWMutex- subscribers map[int]*activitySubscriber- nextSubID int+ db *database.DB+ pubsub *pubsub.Service+ slots *slots.Service+ sweeps *sweeps.Service// running maps an active activity ID to the cancel function of its work// context, so cancellation requests can interrupt in-flight work. Entries@@ -78,149 +59,16 @@ type ActivityService struct {runningMu sync.Mutexrunning map[string]context.CancelCauseFunc- // limiter bounds concurrent queue-opted activities per environment.- // slotReleases maps an activity ID to the release func of the slot it- // holds; the slot is freed when the activity completes.- limiter *activitySlotLimiter- slotMu sync.Mutex- slotReleases map[string]func()-- // terminalPublished records when each activity's terminal snapshot was- // published, so later non-terminal snapshots from slower goroutines can- // be dropped (see admitActivityPublishInternal). Entries are pruned on- // terminal publishes once older than terminalPublishRetention.- terminalPublishedMu sync.Mutex- terminalPublished map[string]time.Time-// publishLocks serializes the commit→publish window of the non-terminal// snapshot writers (AppendMessages, UpdateActivity) per activity, so their// events reach subscribers in commit order — without it, a batch that// commits first but publishes second would revert progress/step/status// fields a concurrent update already streamed. Terminal snapshots are- // ordered separately by admitActivityPublishInternal. Striped by activity+ // ordered separately by the pubsub terminal latch. Striped by activity// ID hash so no per-activity lifecycle management is needed.publishLocks [64]sync.Mutex}-// activitySubscriber buffers stream events between publishers and one stream-// consumer. "activity" events are coalesced in place per activity ID (only the-// latest pending state matters to the UI), so bulk operations emitting rapid-// progress updates cannot overflow the subscriber and force full-snapshot-// resends. Other events keep arrival order in a FIFO bounded by-// subscriberMessageQueueLimit with drop-oldest on overflow.-type activitySubscriber struct {- environmentID string- ch chan activity.StreamEvent- done chan struct{}- wake chan struct{}-- mu sync.Mutex- missed bool- queue []*pendingStreamEvent- pendingActivity map[string]*pendingStreamEvent- messageCount int-}--type pendingStreamEvent struct {- event activity.StreamEvent-}--func newActivitySubscriberInternal(environmentID string, ch chan activity.StreamEvent) *activitySubscriber {- return &activitySubscriber{- environmentID: environmentID,- ch: ch,- done: make(chan struct{}),- wake: make(chan struct{}, 1),- pendingActivity: map[string]*pendingStreamEvent{},- }-}--func isCoalescableEventInternal(event activity.StreamEvent) bool {- return event.Type == "activity" && event.ActivityID != ""-}--func (sub *activitySubscriber) enqueue(event activity.StreamEvent) {- sub.mu.Lock()- if isCoalescableEventInternal(event) {- if pending, ok := sub.pendingActivity[event.ActivityID]; ok {- pending.event = event- sub.mu.Unlock()- return- }- } else {- if sub.messageCount >= subscriberMessageQueueLimit {- sub.dropOldestMessageLockedInternal()- }- sub.messageCount++- }- entry := &pendingStreamEvent{event: event}- sub.queue = append(sub.queue, entry)- if isCoalescableEventInternal(event) {- sub.pendingActivity[event.ActivityID] = entry- }- sub.mu.Unlock()-- select {- case sub.wake <- struct{}{}:- default:- }-}--func (sub *activitySubscriber) dropOldestMessageLockedInternal() {- for i, entry := range sub.queue {- if isCoalescableEventInternal(entry.event) {- continue- }- sub.queue = append(sub.queue[:i], sub.queue[i+1:]...)- sub.messageCount--- sub.missed = true- ctx := context.Background() //nolint:forbidigo // Subscriber buffer trimming runs outside any request context.- slog.WarnContext(ctx, "activity subscriber message buffer full; snapshot will be sent on next heartbeat",- "environmentId", sub.environmentID)- return- }-}--func (sub *activitySubscriber) nextInternal() mo.Option[activity.StreamEvent] {- sub.mu.Lock()- defer sub.mu.Unlock()-- if len(sub.queue) == 0 {- return mo.None[activity.StreamEvent]()- }- entry := sub.queue[0]- sub.queue = sub.queue[1:]- event := entry.event- if isCoalescableEventInternal(event) {- if sub.pendingActivity[event.ActivityID] == entry {- delete(sub.pendingActivity, event.ActivityID)- }- } else {- sub.messageCount--- }- return mo.Some(event)-}--func (sub *activitySubscriber) pump() {- defer close(sub.ch)- for {- event, ok := sub.nextInternal().Get()- if !ok {- select {- case <-sub.wake:- continue- case <-sub.done:- return- }- }- select {- case sub.ch <- event:- case <-sub.done:- return- }- }-}-type (StartActivityRequest = activitylib.StartRequestUpdateActivityRequest = activitylib.UpdateRequest@@ -229,12 +77,11 @@ type (func NewActivityService(db *database.DB, settingsService *settings.SettingsService) *ActivityService {return &ActivityService{- db: db,- subscribers: map[int]*activitySubscriber{},- running: map[string]context.CancelCauseFunc{},- limiter: newActivitySlotLimiterInternal(settingsService),- slotReleases: map[string]func(){},- terminalPublished: map[string]time.Time{},+ db: db,+ pubsub: pubsub.NewService(),+ slots: slots.NewService(settingsService),+ sweeps: sweeps.NewService(db),+ running: map[string]context.CancelCauseFunc{},}}@@ -281,27 +128,7 @@ func (s *ActivityService) RequestCancel(activityID string) bool {return true}-// releaseCancelInternal removes and cancels the registration for activityID.-// Cancelling with a nil cause is a no-op if the context was already cancelled-// (the first cause wins), so a prior ErrCanceled cause is preserved.-func (s *ActivityService) releaseCancelInternal(activityID string) {- activityID = strings.TrimSpace(activityID)- if s == nil || activityID == "" {- return- }-- s.runningMu.Lock()- cancel, ok := s.running[activityID]- if ok {- delete(s.running, activityID)- }- s.runningMu.Unlock()- if ok {- cancel(nil)- }-}--func (s *ActivityService) checkInitInternal() error {+func (s *ActivityService) checkInit() error {if s == nil || s.db == nil {return errors.New("activity service not initialized")}@@ -309,7 +136,7 @@ func (s *ActivityService) checkInitInternal() error {}func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRequest) (*activity.Activity, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, err}@@ -320,7 +147,7 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRereturn nil, fmt.Errorf("failed to load activity: %w", err)}if existing.ID != "" {- dto := activityToDTOInternal(&existing)+ dto := activityToDTO(&existing)return &dto, nil}}@@ -328,22 +155,6 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRenow := time.Now()environmentID := cmp.Or(strings.TrimSpace(req.EnvironmentID), "0")- var startedByUserID, startedByUsername, startedByDisplayName *string- if req.StartedBy != nil {- startedByUserID = mo.EmptyableToOption(strings.TrimSpace(req.StartedBy.ID)).ToPointer()- startedByUsername = mo.EmptyableToOption(strings.TrimSpace(req.StartedBy.Username)).ToPointer()- if req.StartedBy.DisplayName != nil {- startedByDisplayName = mo.EmptyableToOption(strings.TrimSpace(*req.StartedBy.DisplayName)).ToPointer()- }- }-- batchID := req.BatchID- if batchID == nil {- if contextBatchID := utils.ActivityBatchIDFromContext(ctx); contextBatchID != "" {- batchID = &contextBatchID- }- }-// Queue-opted activities take a concurrency slot up front when one is// free; otherwise they are created as queued and AwaitActivitySlot blocks// until a slot opens.@@ -352,35 +163,33 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityReif req.Queue {if req.DeferSlot {status = activity.StatusQueued- } else if release, ok := s.limiter.tryAcquireInternal(ctx, environmentID).Get(); ok {+ } else if release, ok := s.slots.TryAcquire(ctx, environmentID).Get(); ok {slotRelease = release} else {status = activity.StatusQueued}}+ startedBy := kit.FromPtr(req.StartedBy)model := &Activity{+ ID: req.ID,EnvironmentID: environmentID,- BatchID: copyPtrInternal(batchID),- Type: req.Type,+ BatchID: copyPtr(kit.Ternary(req.BatchID != nil, req.BatchID, mo.EmptyableToOption(utils.ActivityBatchIDFromContext(ctx)).ToPointer())),+ Type: cmp.Or(req.Type, activity.TypeAutoUpdate),Status: status,- ResourceType: copyPtrInternal(req.ResourceType),- ResourceID: copyPtrInternal(req.ResourceID),- ResourceName: copyPtrInternal(req.ResourceName),- StartedByUserID: startedByUserID,- StartedByUsername: startedByUsername,- StartedByDisplayName: startedByDisplayName,- Progress: clampProgressPtrInternal(req.Progress),+ ResourceType: copyPtr(req.ResourceType),+ ResourceID: copyPtr(req.ResourceID),+ ResourceName: copyPtr(req.ResourceName),+ StartedByUserID: mo.EmptyableToOption(strings.TrimSpace(startedBy.ID)).ToPointer(),+ StartedByUsername: mo.EmptyableToOption(strings.TrimSpace(startedBy.Username)).ToPointer(),+ StartedByDisplayName: mo.EmptyableToOption(strings.TrimSpace(kit.FromPtr(startedBy.DisplayName))).ToPointer(),+ Progress: clampProgressPtr(req.Progress),Step: strings.TrimSpace(req.Step),LatestMessage: strings.TrimSpace(req.LatestMessage),StartedAt: now,- Metadata: cloneJSONInternal(req.Metadata),+ Metadata: cloneJSON(req.Metadata),CreatedAt: now,}- model.ID = req.ID- if model.Type == "" {- model.Type = activity.TypeAutoUpdate- }create := s.db.WithContext(ctx)if req.ID != "" {@@ -399,22 +208,22 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityReif err := s.db.WithContext(ctx).First(&existing, "id = ?", req.ID).Error; err != nil {return nil, fmt.Errorf("failed to load activity: %w", err)}- dto := activityToDTOInternal(&existing)+ dto := activityToDTO(&existing)return &dto, nil}if slotRelease != nil {- s.registerSlotReleaseInternal(model.ID, slotRelease)+ s.slots.Register(model.ID, slotRelease)}- dto := activityToDTOInternal(model)- s.publishActivityInternal(dto)+ dto := activityToDTO(model)+ s.pubsub.PublishActivity(dto)return &dto, nil}// StartTrackedActivity holds the cancellation lock across persistence and// registration, so cancellation cannot treat the new row as untracked.func (s *ActivityService) StartTrackedActivity(ctx context.Context, req StartActivityRequest) (*activity.Activity, context.Context, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, nil, err}s.runningMu.Lock()@@ -431,37 +240,13 @@ func (s *ActivityService) StartTrackedActivity(ctx context.Context, req StartActreturn started, workCtx, nil}-func (s *ActivityService) registerSlotReleaseInternal(activityID string, release func()) {- s.slotMu.Lock()- if existing, ok := s.slotReleases[activityID]; ok {- existing()- }- s.slotReleases[activityID] = release- s.slotMu.Unlock()-}--func (s *ActivityService) releaseSlotInternal(activityID string) {- if s == nil {- return- }- s.slotMu.Lock()- release, ok := s.slotReleases[activityID]- if ok {- delete(s.slotReleases, activityID)- }- s.slotMu.Unlock()- if ok {- release()- }-}-// AwaitActivitySlot blocks until the queued activity holds a concurrency slot,// then flips its status to running. It returns immediately when the activity// already took a slot at creation. On cancellation the context cause is// returned and the activity stays queued for its caller to finalize.// Implements activitylib.SlotWaiter.func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, environmentID string) error {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return err}activityID = strings.TrimSpace(activityID)@@ -470,18 +255,10 @@ func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, env}environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")- s.slotMu.Lock()- _, held := s.slotReleases[activityID]- s.slotMu.Unlock()- if held {- return nil- }-- release, err := s.limiter.acquireInternal(ctx, environmentID)- if err != nil {+ acquired, err := s.slots.Await(ctx, activityID, environmentID)+ if err != nil || !acquired {return err}- s.registerSlotReleaseInternal(activityID, release)if _, updateErr := s.UpdateActivity(ctx, activityID, UpdateActivityRequest{Status: activity.StatusRunning}); updateErr != nil {slog.WarnContext(ctx, "failed to mark queued activity running", "activityId", activityID, "error", updateErr)@@ -506,7 +283,7 @@ func (s *ActivityService) AwaitActivitySlotBounded(ctx context.Context, activity}func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string, req UpdateActivityRequest) (*activity.Activity, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, err}activityID = strings.TrimSpace(activityID)@@ -524,7 +301,7 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,updates["resource_name"] = strings.TrimSpace(*req.ResourceName)}if req.Progress != nil {- updates["progress"] = *clampProgressPtrInternal(req.Progress)+ updates["progress"] = *clampProgressPtr(req.Progress)}if req.Step != nil {updates["step"] = strings.TrimSpace(*req.Step)@@ -536,13 +313,13 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,updates["error"] = strings.TrimSpace(*req.Error)}if req.Metadata != nil {- updates["metadata"] = cloneJSONInternal(req.Metadata)+ updates["metadata"] = cloneJSON(req.Metadata)}// Hold the per-activity publish lock across commit and publish so this// snapshot cannot be overtaken on the stream by a concurrent append batch// that committed earlier but would publish later.- lock := s.publishLockInternal(activityID)+ lock := s.publishLock(activityID)lock.Lock()defer lock.Unlock()@@ -563,8 +340,8 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,return nil, err}- dto := activityToDTOInternal(&model)- s.publishActivityInternal(dto)+ dto := activityToDTO(&model)+ s.pubsub.PublishActivity(dto)return &dto, nil}@@ -584,9 +361,9 @@ func (s *ActivityService) AppendMessage(ctx context.Context, activityID string,// per batch, not per line) so it reflects lifecycle fields a concurrent// CompleteActivity/UpdateActivity committed first; a terminal write that// commits after this transaction but publishes before this snapshot is-// handled by admitActivityPublishInternal dropping the stale event.+// handled by the pubsub terminal latch dropping the stale event.func (s *ActivityService) AppendMessages(ctx context.Context, activityID string, reqs []AppendActivityMessageRequest) ([]activity.Message, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, err}activityID = strings.TrimSpace(activityID)@@ -594,7 +371,7 @@ func (s *ActivityService) AppendMessages(ctx context.Context, activityID string,return nil, errors.New("activity id is required")}- messages, updates := buildAppendBatchInternal(activityID, reqs)+ messages, updates := buildAppendBatch(activityID, reqs)if len(messages) == 0 {return nil, nil}@@ -603,29 +380,58 @@ func (s *ActivityService) AppendMessages(ctx context.Context, activityID string,// batch's snapshot cannot be overtaken on the stream by a concurrent// UpdateActivity that committed later — subscribers must see the two// snapshots in commit order.- lock := s.publishLockInternal(activityID)+ lock := s.publishLock(activityID)lock.Lock()defer lock.Unlock()- current, err := s.appendBatchWithRetryInternal(ctx, activityID, messages, updates)- if err != nil {+ // The Writer drain cannot replay a lost batch, so retry like CompleteActivity.+ var current Activity+ if err := s.retryWrite(ctx, activityID, "message batch", func(tx *gorm.DB) error {+ if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return errors.New("activity not found")+ }+ return fmt.Errorf("failed to load activity: %w", err)+ }+ if err := tx.Create(&messages).Error; err != nil {+ return fmt.Errorf("failed to append activity message: %w", err)+ }+ result := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates)+ if result.Error != nil {+ return fmt.Errorf("failed to update activity latest message: %w", result.Error)+ }+ if result.RowsAffected == 0 {+ return errors.New("activity not found")+ }+ if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("failed to load updated activity: %w", err)+ }+ return nil+ }); err != nil {return nil, err}out := make([]activity.Message, 0, len(messages))for _, message := range messages {- dto := activityMessageToDTOInternal(message)- s.publishMessageInternal(current.EnvironmentID, current.Type, dto)+ dto := activity.Message{+ ID: message.ID,+ ActivityID: message.ActivityID,+ Level: message.Level,+ Message: message.Message,+ Payload: cloneJSON(message.Payload),+ CreatedAt: message.CreatedAt,+ }+ s.pubsub.PublishMessage(current.EnvironmentID, current.Type, dto)out = append(out, dto)}- s.publishActivityInternal(activityToDTOInternal(current))+ s.pubsub.PublishActivity(activityToDTO(¤t))return out, nil}-// buildAppendBatchInternal turns the request batch into insertable message rows+// buildAppendBatch turns the request batch into insertable message rows// plus the coalesced Activity column updates (latest message/progress/step from// the last request carrying each).-func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageRequest) ([]*ActivityMessage, map[string]any) {+func buildAppendBatch(activityID string, reqs []AppendActivityMessageRequest) ([]*ActivityMessage, map[string]any) {now := time.Now()messages := make([]*ActivityMessage, 0, len(reqs))latestMessage := ""@@ -646,7 +452,7 @@ func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageReqActivityID: activityID,Level: level,Message: messageText,- Payload: cloneJSONInternal(req.Payload),+ Payload: cloneJSON(req.Payload),// Spread inside the shared timestamp so the created_at sort// used for retrieval keeps the original line order.CreatedAt: now.Add(time.Duration(len(messages)) * time.Microsecond),@@ -669,7 +475,7 @@ func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageReq"updated_at": now,}if latestProgress != nil {- updates["progress"] = *clampProgressPtrInternal(latestProgress)+ updates["progress"] = *clampProgressPtr(latestProgress)}if latestStep != "" {updates["step"] = latestStep@@ -677,70 +483,25 @@ func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageReqreturn messages, updates}-// appendBatchWithRetryInternal commits the batch INSERT plus Activity update in-// one transaction. Bulk appends contend with terminal-status writes for the-// SQLite write lock, and a batch lost to transient SQLITE_BUSY silently drops-// up to 32 output lines — the Writer drain has no error path to replay them —-// so retry like CompleteActivity does.-func (s *ActivityService) appendBatchWithRetryInternal(ctx context.Context, activityID string, messages []*ActivityMessage, updates map[string]any) (*Activity, error) {- var current Activity- const appendWriteAttempts = 3- var writeErr error- for attempt := 1; attempt <= appendWriteAttempts; attempt++ {- writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {- if errors.Is(err, gorm.ErrRecordNotFound) {- return errors.New("activity not found")- }- return fmt.Errorf("failed to load activity: %w", err)- }-- if err := tx.Create(&messages).Error; err != nil {- return fmt.Errorf("failed to append activity message: %w", err)- }-- result := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates)- if result.Error != nil {- return fmt.Errorf("failed to update activity latest message: %w", result.Error)- }- if result.RowsAffected == 0 {- return errors.New("activity not found")- }- if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {- return fmt.Errorf("failed to load updated activity: %w", err)- }- return nil- })- if writeErr == nil {- return ¤t, nil- }- if attempt == appendWriteAttempts || !dbutil.IsRetryableWriteError(writeErr) {- return nil, writeErr+// retryWrite runs a write transaction, retrying transient SQLite contention+// (SQLITE_BUSY under bulk activity writes) up to three times.+func (s *ActivityService) retryWrite(ctx context.Context, activityID, what string, write func(tx *gorm.DB) error) error {+ const attempts = 3+ for attempt := 1; ; attempt++ {+ err := s.db.WithContext(ctx).Transaction(write)+ if err == nil || attempt == attempts || !dbutil.IsRetryableWriteError(err) {+ return err}- slog.WarnContext(ctx, "retrying activity message batch write", "activityId", activityID, "attempt", attempt, "error", writeErr)+ slog.WarnContext(ctx, "retrying activity write", "write", what, "activityId", activityID, "attempt", attempt, "error", err)time.Sleep(250 * time.Millisecond * time.Duration(attempt))}- return nil, writeErr}-func (- s *ActivityService,-) CompleteActivity(- ctx context.Context,- activityID string,- status activity.Status,- finalMessage string,- errMessage *string,- finalStep ...string,-) (- *activity.Activity,- error,-) {- if err := s.checkInitInternal(); err != nil {+func (s *ActivityService) CompleteActivity(ctx context.Context, activityID string, status activity.Status, finalMessage string, errMessage *string, finalStep ...string) (*activity.Activity, error) {+ if err := s.checkInit(); err != nil {return nil, err}- status = cmp.Or(status, activity.StatusSuccess)- if status != activity.StatusSuccess && status != activity.StatusFailed && status != activity.StatusCancelled {+ if !slices.Contains(terminalActivityStatuses, status) {status = activity.StatusSuccess}@@ -750,9 +511,15 @@ func (}// The activity is reaching a terminal state; release any cancel- // registration and free its concurrency slot.- s.releaseCancelInternal(activityID)- s.releaseSlotInternal(activityID)+ // registration (a nil cause keeps an earlier ErrCanceled) and free its slot.+ s.runningMu.Lock()+ cancel, tracked := s.running[activityID]+ delete(s.running, activityID)+ s.runningMu.Unlock()+ if tracked {+ cancel(nil)+ }+ s.slots.Release(activityID)// Detach from cancellation so the terminal write always lands — completion is// often triggered precisely because the work context was cancelled.@@ -760,37 +527,22 @@ func (now := time.Now()var model Activity- // A lost terminal write leaves the activity stuck in running forever, so- // retry transient DB contention (SQLITE_BUSY under bulk activity writes)- // instead of surfacing it once and giving up.- const completeWriteAttempts = 3- var writeErr error- for attempt := 1; attempt <= completeWriteAttempts; attempt++ {- writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {- return fmt.Errorf("failed to load activity: %w", err)- }+ // A lost terminal write leaves the activity stuck in running forever.+ if err := s.retryWrite(ctx, activityID, "terminal status", func(tx *gorm.DB) error {+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("failed to load activity: %w", err)+ }- updates := completeActivityUpdatesInternal(model.StartedAt, status, finalMessage, errMessage, finalStep, now)- if err := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {- return fmt.Errorf("failed to complete activity: %w", err)- }- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {- return fmt.Errorf("failed to load completed activity: %w", err)- }- return nil- })- if writeErr == nil {- break+ updates := completeActivityUpdates(model.StartedAt, status, finalMessage, errMessage, finalStep, now)+ if err := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {+ return fmt.Errorf("failed to complete activity: %w", err)}- if attempt == completeWriteAttempts || !dbutil.IsRetryableWriteError(writeErr) {- return nil, writeErr+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {+ return fmt.Errorf("failed to load completed activity: %w", err)}- slog.WarnContext(ctx, "retrying activity terminal status write", "activityId", activityID, "attempt", attempt, "error", writeErr)- time.Sleep(250 * time.Millisecond * time.Duration(attempt))- }- if writeErr != nil {- return nil, writeErr+ return nil+ }); err != nil {+ return nil, err}if strings.TrimSpace(finalMessage) != "" {@@ -811,11 +563,11 @@ func (}}- dto := s.publishTerminalSnapshotInternal(ctx, &model)+ dto := s.publishTerminalSnapshot(ctx, &model)return &dto, nil}-// publishTerminalSnapshotInternal publishes an activity's terminal event from+// publishTerminalSnapshot publishes an activity's terminal event from// a snapshot re-read under the per-activity publish lock. Terminal writers// cannot hold that lock across their own transactions (CompleteActivity// appends its final message through the locked AppendMessages path), so a@@ -824,8 +576,8 @@ func (// guarantees the terminal event carries every field already streamed. The// passed model is published as-is if the re-read fails, and is updated in// place otherwise so callers return the published state.-func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, model *Activity) activity.Activity {- lock := s.publishLockInternal(model.ID)+func (s *ActivityService) publishTerminalSnapshot(ctx context.Context, model *Activity) activity.Activity {+ lock := s.publishLock(model.ID)lock.Lock()defer lock.Unlock()@@ -835,8 +587,8 @@ func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, m} else {*model = fresh}- dto := activityToDTOInternal(model)- s.publishActivityInternal(dto)+ dto := activityToDTO(model)+ s.pubsub.PublishActivity(dto)return dto}@@ -846,7 +598,7 @@ func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, m// but only if it is still active. Returns ErrActivityNotCancelable if the activity// has already reached a terminal state, or gorm.ErrRecordNotFound if it is unknown.func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, activityID, requestedBy string) (*activity.Activity, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, err}activityID = strings.TrimSpace(activityID)@@ -883,7 +635,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act// terminal status, which reaches clients via the activity stream. Return// the pre-cancel snapshot rather than reloading here: the worker has not// finished unwinding yet, so a reload would still report "running".- return new(activityToDTOInternal(&model)), nil+ return new(activityToDTO(&model)), nil}// Untracked work (e.g. after a process restart, or a queued activity with no@@ -895,7 +647,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, actif err := tx.First(&finalized, "id = ? AND environment_id = ?", activityID, environmentID).Error; err != nil {return err}- updates := completeActivityUpdatesInternal(finalized.StartedAt, activity.StatusCancelled, cancelledMessageInternal, nil, nil, now)+ updates := completeActivityUpdates(finalized.StartedAt, activity.StatusCancelled, cancelledMessage, nil, nil, now)result := tx.Model(&Activity{}).Where("id = ? AND status IN ?", activityID, []activity.Status{activity.StatusQueued, activity.StatusRunning}).Updates(updates)@@ -913,7 +665,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, actreturn nil, err}- dto := s.publishTerminalSnapshotInternal(writeCtx, &finalized)+ dto := s.publishTerminalSnapshot(writeCtx, &finalized)return &dto, nil}@@ -921,40 +673,40 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act// across a prior process lifetime as failed. It intentionally scopes cleanup to// old image-update-check activities so startup repair cannot affect other work.func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64, error) {- if s == nil || s.db == nil {- return 0, nil- }+ return s.finalizeSwept(ctx, "stale image update check",+ func() ([]sweeps.Candidate, error) { return s.sweeps.StaleImageUpdateChecks(ctx) },+ func(sweeps.Candidate) (activity.Status, string) {+ return activity.StatusFailed, "Image update check failed because it was stale after Arcane restarted"+ },+ "Image update check failed",+ )+}- cutoff := time.Now().Add(-staleImageUpdateCheckAge)- var staleChecks []Activity- if err := s.db.WithContext(ctx).- Where("type = ? AND status = ? AND started_at < ?", activity.TypeImageUpdateCheck, activity.StatusRunning, cutoff).- Find(&staleChecks).Error; err != nil {- return 0, fmt.Errorf("find stale image update checks: %w", err)- }-- const message = "Image update check failed because it was stale after Arcane restarted"- errMessage := message- var failed int64- var failErrs []error- for i := range staleChecks {- if _, err := s.CompleteActivity(ctx, staleChecks[i].ID, activity.StatusFailed, message, &errMessage, "Image update check failed"); err != nil {- failErrs = append(failErrs, fmt.Errorf("fail stale image update check %s: %w", staleChecks[i].ID, err))+// finalizeSwept completes each swept candidate with the status and message+// resolve picks; failed ones also record the message as their error.+func (s *ActivityService) finalizeSwept(+ ctx context.Context,+ label string,+ find func() ([]sweeps.Candidate, error),+ resolve func(sweeps.Candidate) (activity.Status, string),+ finalStep ...string,+) (int64, error) {+ candidates, err := find()+ if err != nil {+ return 0, fmt.Errorf("find %s activities: %w", label, err)+ }+ var finalized int64+ var errs []error+ for _, candidate := range candidates {+ status, message := resolve(candidate)+ errMessage := kit.Ternary(status == activity.StatusFailed, new(message), nil)+ if _, completeErr := s.CompleteActivity(ctx, candidate.ID, status, message, errMessage, finalStep...); completeErr != nil {+ errs = append(errs, fmt.Errorf("finalize %s activity %s: %w", label, candidate.ID, completeErr))continue}- failed+++ finalized++}-- return failed, errors.Join(failErrs...)-}--// isTrackedInternal reports whether activityID has a live work registration in-// this process (i.e. a worker goroutine called Track and has not completed yet).-func (s *ActivityService) isTrackedInternal(activityID string) bool {- s.runningMu.Lock()- defer s.runningMu.Unlock()- _, ok := s.running[activityID]- return ok+ return finalized, errors.Join(errs...)}// FailAbandonedActivities marks queued/running activities whose worker is no@@ -970,27 +722,21 @@ func (s *ActivityService) isTrackedInternal(activityID string) bool {// fallback: a worker completing concurrently wins the race and the row is// skipped here.func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, error) {- if s == nil || s.db == nil {- return 0, nil- }-activeStatuses := []activity.Status{activity.StatusQueued, activity.StatusRunning}s.runningMu.Lock()trackedIDs := slices.Collect(maps.Keys(s.running))s.runningMu.Unlock()- query := s.db.WithContext(ctx).- Select("id", "started_at").- Where("status IN ? AND started_at < ?", activeStatuses, time.Now().Add(-abandonedActivityGrace)).- Where("type <> ?", activity.TypeJobRun)- if len(trackedIDs) > 0 {- query = query.Where("id NOT IN ?", trackedIDs)- }- var candidates []Activity- if err := query.Order("started_at").Limit(abandonedActivitySweepLimit).Find(&candidates).Error; err != nil {- return 0, fmt.Errorf("find abandoned activities: %w", err)+ candidates, findErr := s.sweeps.Abandoned(ctx, trackedIDs)+ if findErr != nil {+ return 0, fmt.Errorf("find abandoned activities: %w", findErr)}// A worker may have Tracked its activity since the snapshot above.- candidates = slices.DeleteFunc(candidates, func(candidate Activity) bool { return s.isTrackedInternal(candidate.ID) })+ s.runningMu.Lock()+ candidates = slices.DeleteFunc(candidates, func(candidate sweeps.Candidate) bool {+ _, tracked := s.running[candidate.ID]+ return tracked+ })+ s.runningMu.Unlock()if len(candidates) == 0 {return 0, nil}@@ -1007,7 +753,7 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, edurationArgs = append(durationArgs, candidate.ID, now.Sub(candidate.StartedAt).Milliseconds())}durationExpr.WriteString(" ELSE duration_ms END")- updates := completeActivityUpdatesInternal(now, activity.StatusFailed, message, new(message), nil, now)+ updates := completeActivityUpdates(now, activity.StatusFailed, message, new(message), nil, now)updates["duration_ms"] = gorm.Expr(durationExpr.String(), durationArgs...)// Status-guarded so a worker completing concurrently wins; only rows this@@ -1028,8 +774,8 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e}for i := range finalized {- s.releaseSlotInternal(finalized[i].ID)- s.publishTerminalSnapshotInternal(ctx, &finalized[i])+ s.slots.Release(finalized[i].ID)+ s.publishTerminalSnapshot(ctx, &finalized[i])}return int64(len(finalized)), nil@@ -1039,38 +785,18 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e// Queued state is owned by a live goroutine blocked on AwaitActivitySlot, so a// queued row after a restart can never start running.func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context, protectedIDs ...string) (int64, error) {- if s == nil || s.db == nil {- return 0, nil- }-- var queued []Activity- if err := s.db.WithContext(ctx).- Where("status = ?", activity.StatusQueued).- Where("type <> ?", activity.TypeJobRun).- Where("id NOT IN ?", append(protectedIDs, "")).- Find(&queued).Error; err != nil {- return 0, fmt.Errorf("find orphaned queued activities: %w", err)- }-- const message = "Queued activity was interrupted by an Arcane restart"- errMessage := message- var failed int64- var failErrs []error- for i := range queued {- if _, err := s.CompleteActivity(ctx, queued[i].ID, activity.StatusFailed, message, &errMessage); err != nil {- failErrs = append(failErrs, fmt.Errorf("fail orphaned queued activity %s: %w", queued[i].ID, err))- continue- }- failed++- }-- return failed, errors.Join(failErrs...)+ return s.finalizeSwept(ctx, "orphaned queued",+ func() ([]sweeps.Candidate, error) { return s.sweeps.OrphanedQueued(ctx, protectedIDs) },+ func(sweeps.Candidate) (activity.Status, string) {+ return activity.StatusFailed, "Queued activity was interrupted by an Arcane restart"+ },+ )}// PatchActivityMetadata merges patch into the activity's existing metadata,// unlike UpdateActivity which replaces the metadata wholesale.func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID string, patch database.JSON) error {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return err}activityID = strings.TrimSpace(activityID)@@ -1086,7 +812,7 @@ func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityIDif err := tx.First(&record, "id = ?", activityID).Error; err != nil {return fmt.Errorf("failed to load activity: %w", err)}- merged := cloneJSONInternal(record.Metadata)+ merged := cloneJSON(record.Metadata)if merged == nil {merged = database.JSON{}}@@ -1104,41 +830,18 @@ func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID// self-update completed by restarting Arcane, so it is recorded as success;// anything else still running at startup was interrupted and is failed.func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context, protectedIDs ...string) (int64, error) {- if s == nil || s.db == nil {- return 0, nil- }-- var stale []Activity- if err := s.db.WithContext(ctx).- Where("type = ? AND status = ?", activity.TypeAutoUpdate, activity.StatusRunning).- Where("id NOT IN ?", append(protectedIDs, "")).- Find(&stale).Error; err != nil {- return 0, fmt.Errorf("find stale auto-update activities: %w", err)- }-- var resolved int64- var resolveErrs []error- for i := range stale {- status := activity.StatusFailed- message := "Auto-update interrupted by Arcane restart"- var errMessage *string- if selfUpdate, _ := stale[i].Metadata["selfUpdateTriggered"].(bool); selfUpdate {- status = activity.StatusSuccess- message = "Auto-update completed — Arcane restarted with the updated image"- } else {- errMessage = new(message)- }- if _, err := s.CompleteActivity(ctx, stale[i].ID, status, message, errMessage); err != nil {- resolveErrs = append(resolveErrs, fmt.Errorf("resolve stale auto-update activity %s: %w", stale[i].ID, err))- continue- }- resolved++- }-- return resolved, errors.Join(resolveErrs...)+ return s.finalizeSwept(ctx, "stale auto-update",+ func() ([]sweeps.Candidate, error) { return s.sweeps.StaleAutoUpdates(ctx, protectedIDs) },+ func(candidate sweeps.Candidate) (activity.Status, string) {+ if selfUpdate, _ := candidate.Metadata["selfUpdateTriggered"].(bool); selfUpdate {+ return activity.StatusSuccess, "Auto-update completed — Arcane restarted with the updated image"+ }+ return activity.StatusFailed, "Auto-update interrupted by Arcane restart"+ },+ )}-func completeActivityUpdatesInternal(startedAt time.Time, status activity.Status, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {+func completeActivityUpdates(startedAt time.Time, status activity.Status, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {updates := map[string]any{"status": status,"ended_at": now,@@ -1163,7 +866,7 @@ func completeActivityUpdatesInternal(startedAt time.Time, status activity.Status}func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]activity.Activity, pagination.Response, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, pagination.Response{}, err}@@ -1171,7 +874,7 @@ func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environmevar activities []Activityq := s.db.WithContext(ctx).Model(&Activity{}).Where("environment_id = ?", environmentID)- q = scopeJobActivityVisibilityInternal(ctx, q, environmentID, authz.PermActivitiesRead)+ q = scopeJobActivityVisibility(ctx, q, environmentID, authz.PermActivitiesRead)if term := strings.TrimSpace(params.Search); term != "" {escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(term)@@ -1204,13 +907,13 @@ func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environmeout := make([]activity.Activity, 0, len(activities))for i := range activities {- out = append(out, activityToDTOInternal(&activities[i]))+ out = append(out, activityToDTO(&activities[i]))}return out, paginationResp, nil}func (s *ActivityService) GetActivityDetail(ctx context.Context, environmentID, activityID string, limit int) (*activity.Detail, error) {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return nil, err}if limit <= 0 || limit > defaultActivityMessages {@@ -1234,20 +937,24 @@ func (s *ActivityService) GetActivityDetail(ctx context.Context, environmentID,}outMessages := make([]activity.Message, 0, len(messages))- for _, v := range slices.Backward(messages) {- outMessages = append(outMessages, activityMessageToDTOInternal(&v))+ for _, message := range slices.Backward(messages) {+ outMessages = append(outMessages, activity.Message{+ ID: message.ID,+ ActivityID: message.ActivityID,+ Level: message.Level,+ Message: message.Message,+ Payload: cloneJSON(message.Payload),+ CreatedAt: message.CreatedAt,+ })}return &activity.Detail{- Activity: activityToDTOInternal(&model),+ Activity: activityToDTO(&model),Messages: outMessages,}, nil}func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEntries int) (int64, error) {- if s == nil || s.db == nil {- return 0, nil- }if retentionDays < 0 {retentionDays = defaultActivityRetentionDays}@@ -1259,11 +966,13 @@ func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEnif err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {if retentionDays > 0 {cutoff := time.Now().Add(-time.Duration(retentionDays) * 24 * time.Hour)- ids, err := findTerminalActivityIDsInternal(tx.Where("COALESCE(ended_at, updated_at, created_at) < ?", cutoff))- if err != nil {+ var ids []string+ if err := tx.Model(&Activity{}).+ Where("status IN ? AND COALESCE(ended_at, updated_at, created_at) < ?", terminalActivityStatuses, cutoff).+ Pluck("id", &ids).Error; err != nil {return fmt.Errorf("failed to find activities older than retention window: %w", err)}- count, err := deleteActivitiesByIDInternal(tx, ids)+ count, err := deleteActivitiesByID(tx, ids)if err != nil {return err}@@ -1271,11 +980,23 @@ func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEn}if maxEntries > 0 {- ids, err := findActivityIDsBeyondHistoryLimitInternal(tx, maxEntries)- if err != nil {- return err+ var ids []string+ if err := tx.Raw(`+ SELECT ranked.id+ FROM (+ SELECT id,+ ROW_NUMBER() OVER (+ PARTITION BY environment_id+ ORDER BY COALESCE(ended_at, updated_at, created_at) DESC, started_at DESC+ ) AS activity_rank+ FROM activities+ WHERE status IN ?+ ) ranked+ WHERE ranked.activity_rank > ?+ `, terminalActivityStatuses, maxEntries).Scan(&ids).Error; err != nil {+ return fmt.Errorf("failed to find excess activities: %w", err)}- count, err := deleteActivitiesByIDInternal(tx, ids)+ count, err := deleteActivitiesByID(tx, ids)if err != nil {return err}@@ -1291,19 +1012,16 @@ func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEn}func (s *ActivityService) DeleteHistory(ctx context.Context, environmentID string) (int64, error) {- if s == nil || s.db == nil {- return 0, nil- }-environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")var deleted int64if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- ids, err := findTerminalActivityIDsInternal(scopeJobActivityVisibilityInternal(ctx, tx.Where("environment_id = ?", environmentID), environmentID, authz.PermActivitiesDelete))- if err != nil {+ var ids []string+ query := tx.Model(&Activity{}).Where("environment_id = ? AND status IN ?", environmentID, terminalActivityStatuses)+ if err := scopeJobActivityVisibility(ctx, query, environmentID, authz.PermActivitiesDelete).Pluck("id", &ids).Error; err != nil {return fmt.Errorf("failed to find activity history: %w", err)}- count, err := deleteActivitiesByIDInternal(tx, ids)+ count, err := deleteActivitiesByID(tx, ids)if err != nil {return err}@@ -1316,147 +1034,13 @@ func (s *ActivityService) DeleteHistory(ctx context.Context, environmentID strinreturn deleted, nil}-func (s *ActivityService) Subscribe(environmentID string) (<-chan activity.StreamEvent, func() bool, func()) {- ch := make(chan activity.StreamEvent, 64)- if s == nil {- close(ch)- return ch, func() bool { return false }, func() {}- }-- environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")-- sub := newActivitySubscriberInternal(environmentID, ch)- s.subscribersMu.Lock()- s.nextSubID++- id := s.nextSubID- s.subscribers[id] = sub- s.subscribersMu.Unlock()- go sub.pump()-- missedEvents := func() bool {- s.subscribersMu.RLock()- localSub, ok := s.subscribers[id]- s.subscribersMu.RUnlock()- if !ok {- return false- }-- localSub.mu.Lock()- defer localSub.mu.Unlock()- if !localSub.missed {- return false- }- localSub.missed = false- return true- }-- unsubscribe := func() {- s.subscribersMu.Lock()- localSub, ok := s.subscribers[id]- if ok {- delete(s.subscribers, id)- }- s.subscribersMu.Unlock()- if ok {- // The pump goroutine owns ch and closes it on shutdown.- close(localSub.done)- }- }-- return ch, missedEvents, unsubscribe-}--func (s *ActivityService) publishActivityInternal(item activity.Activity) {- if !s.admitActivityPublishInternal(item) {- return- }- s.publishInternal(item.EnvironmentID, activity.StreamEvent{- Type: "activity",- ActivityID: item.ID,- Activity: &item,- Timestamp: time.Now(),- })-}--func (s *ActivityService) publishLockInternal(activityID string) *sync.Mutex {+func (s *ActivityService) publishLock(activityID string) *sync.Mutex {h := fnv.New32a()_, _ = h.Write([]byte(activityID))return &s.publishLocks[h.Sum32()%uint32(len(s.publishLocks))]}-func isTerminalActivityStatusInternal(status activity.Status) bool {- return status == activity.StatusSuccess ||- status == activity.StatusFailed ||- status == activity.StatusCancelled-}--// admitActivityPublishInternal orders activity events across the commit →-// publish gap. Writes publish their snapshot after committing, so a goroutine-// that committed before a terminal write can publish after it; since a-// terminal activity emits no further events to correct the stream (and-// subscriber coalescing would even replace an undelivered terminal event in-// place), such a stale non-terminal snapshot must be dropped, not delivered.-// Activities never leave a terminal status, so once a terminal snapshot is-// published, any later non-terminal snapshot for that ID is stale by-// construction.-func (s *ActivityService) admitActivityPublishInternal(item activity.Activity) bool {- if s == nil {- return true- }- s.terminalPublishedMu.Lock()- defer s.terminalPublishedMu.Unlock()- if !isTerminalActivityStatusInternal(item.Status) {- _, sealed := s.terminalPublished[item.ID]- return !sealed- }- now := time.Now()- for id, publishedAt := range s.terminalPublished {- if now.Sub(publishedAt) > terminalPublishRetention {- delete(s.terminalPublished, id)- }- }- if s.terminalPublished == nil {- s.terminalPublished = map[string]time.Time{}- }- s.terminalPublished[item.ID] = now- return true-}--func (s *ActivityService) publishMessageInternal(environmentID string, activityType activity.Type, message activity.Message) {- s.publishInternal(environmentID, activity.StreamEvent{- Type: "message",- ActivityID: message.ActivityID,- ActivityType: activityType,- Message: &message,- Timestamp: time.Now(),- })-}--func (s *ActivityService) publishInternal(environmentID string, event activity.StreamEvent) {- if s == nil {- return- }- s.subscribersMu.RLock()- subs := make([]*activitySubscriber, 0, len(s.subscribers))- for _, sub := range s.subscribers {- if sub.environmentID == environmentID {- subs = append(subs, sub)- }- }- s.subscribersMu.RUnlock()-- for _, sub := range subs {- // Source labeling mutates activity fields independently for each subscriber.- subscriberEvent := event- if event.Activity != nil {- subscriberEvent.Activity = new(*event.Activity)- }- subscriberEvent.Activities = slices.Clone(event.Activities)- sub.enqueue(subscriberEvent)- }-}--func activityToDTOInternal(model *Activity) activity.Activity {+func activityToDTO(model *Activity) activity.Activity {if model == nil {return activity.Activity{}}@@ -1464,55 +1048,51 @@ func activityToDTOInternal(model *Activity) activity.Activity {ID: model.ID,EnvironmentID: model.EnvironmentID,SourceEnvironmentID: model.EnvironmentID,- BatchID: copyPtrInternal(model.BatchID),+ BatchID: copyPtr(model.BatchID),Type: model.Type,Status: model.Status,- ResourceType: copyPtrInternal(model.ResourceType),- ResourceID: copyPtrInternal(model.ResourceID),- ResourceName: copyPtrInternal(model.ResourceName),- Progress: clampProgressPtrInternal(model.Progress),+ ResourceType: copyPtr(model.ResourceType),+ ResourceID: copyPtr(model.ResourceID),+ ResourceName: copyPtr(model.ResourceName),+ Progress: clampProgressPtr(model.Progress),Step: model.Step,LatestMessage: model.LatestMessage,- StartedBy: activityStartedByDTOInternal(model),- StartedAt: model.StartedAt,- EndedAt: copyPtrInternal(model.EndedAt),- DurationMs: copyPtrInternal(model.DurationMs),- Error: copyPtrInternal(model.Error),- Metadata: jsonToMapInternal(model.Metadata),- CreatedAt: model.CreatedAt,- UpdatedAt: copyPtrInternal(model.UpdatedAt),- }-}--func activityMessageToDTOInternal(model *ActivityMessage) activity.Message {- if model == nil {- return activity.Message{}- }- return activity.Message{- ID: model.ID,- ActivityID: model.ActivityID,- Level: model.Level,- Message: model.Message,- Payload: jsonToMapInternal(model.Payload),+ StartedBy: func() *activity.StartedBy {+ username := strings.TrimSpace(kit.FromPtr(model.StartedByUsername))+ if username == "" {+ return &activity.StartedBy{Username: "System"}+ }+ return &activity.StartedBy{+ Username: username,+ UserID: strings.TrimSpace(kit.FromPtr(model.StartedByUserID)),+ DisplayName: strings.TrimSpace(kit.FromPtr(model.StartedByDisplayName)),+ }+ }(),+ StartedAt: model.StartedAt,+ EndedAt: copyPtr(model.EndedAt),+ DurationMs: copyPtr(model.DurationMs),+ Error: copyPtr(model.Error),+ Metadata: cloneJSON(model.Metadata),CreatedAt: model.CreatedAt,+ UpdatedAt: copyPtr(model.UpdatedAt),}}-func copyPtrInternal[T any](value *T) *T {+func copyPtr[T any](value *T) *T {if value == nil {return nil}return new(*value)}-func clampProgressPtrInternal(value *int) *int {+func clampProgressPtr(value *int) *int {if value == nil {return nil}return new(min(max(*value, 0), 100))}-func cloneJSONInternal(input database.JSON) database.JSON {+func cloneJSON(input database.JSON) database.JSON {if len(input) == 0 {return nil}@@ -1521,54 +1101,13 @@ func cloneJSONInternal(input database.JSON) database.JSON {return out}-func jsonToMapInternal(input database.JSON) map[string]any {- if len(input) == 0 {- return nil- }- out := make(map[string]any, len(input))- maps.Copy(out, input)- return out-}--func terminalActivityStatusesInternal() []activity.Status {- return []activity.Status{- activity.StatusSuccess,- activity.StatusFailed,- activity.StatusCancelled,- }-}--func findTerminalActivityIDsInternal(q *gorm.DB) ([]string, error) {- var activityIDs []string- if err := q.Model(&Activity{}).- Where("status IN ?", terminalActivityStatusesInternal()).- Pluck("id", &activityIDs).Error; err != nil {- return nil, err- }- return activityIDs, nil-}--func findActivityIDsBeyondHistoryLimitInternal(tx *gorm.DB, maxEntries int) ([]string, error) {- var activityIDs []string- if err := tx.Raw(`- SELECT ranked.id- FROM (- SELECT id,- ROW_NUMBER() OVER (- PARTITION BY environment_id- ORDER BY COALESCE(ended_at, updated_at, created_at) DESC, started_at DESC- ) AS activity_rank- FROM activities- WHERE status IN ?- ) ranked- WHERE ranked.activity_rank > ?- `, terminalActivityStatusesInternal(), maxEntries).Scan(&activityIDs).Error; err != nil {- return nil, fmt.Errorf("failed to find excess activities: %w", err)- }- return activityIDs, nil+var terminalActivityStatuses = []activity.Status{+ activity.StatusSuccess,+ activity.StatusFailed,+ activity.StatusCancelled,}-func deleteActivitiesByIDInternal(tx *gorm.DB, activityIDs []string) (int64, error) {+func deleteActivitiesByID(tx *gorm.DB, activityIDs []string) (int64, error) {if len(activityIDs) == 0 {return 0, nil}@@ -1591,74 +1130,38 @@ func deleteActivitiesByIDInternal(tx *gorm.DB, activityIDs []string) (int64, errreturn totalDeleted, nil}-func activityStartedByDTOInternal(model *Activity) *activity.StartedBy {- if model.StartedByUsername == nil || strings.TrimSpace(*model.StartedByUsername) == "" {- return &activity.StartedBy{Username: "System"}- }-- startedBy := &activity.StartedBy{- Username: strings.TrimSpace(*model.StartedByUsername),- }- if model.StartedByUserID != nil {- startedBy.UserID = strings.TrimSpace(*model.StartedByUserID)- }- if model.StartedByDisplayName != nil {- startedBy.DisplayName = strings.TrimSpace(*model.StartedByDisplayName)- }- return startedBy-}-// FailInterruptedBackups finalizes backup activities before startup admits work.func (s *ActivityService) FailInterruptedBackups(ctx context.Context, protectedIDs ...string) error {- var pending []Activity- if err := s.db.WithContext(- ctx,- ).Where(- "status IN ?",- []activity.Status{- activity.StatusQueued,- activity.StatusRunning,+ _, err := s.finalizeSwept(ctx, "interrupted backup",+ func() ([]sweeps.Candidate, error) {+ pending, err := s.sweeps.InterruptedBackups(ctx, protectedIDs)+ s.runningMu.Lock()+ defer s.runningMu.Unlock()+ return slices.DeleteFunc(pending, func(candidate sweeps.Candidate) bool {+ _, tracked := s.running[candidate.ID]+ return tracked+ }), err},- ).Where(- "id NOT IN ?",- append(- protectedIDs,- "",- ),- ).Find(- &pending,- ).Error; err != nil {- return fmt.Errorf("find interrupted backup activities: %w", err)- }- const message = "Backup interrupted by Arcane restart"- var result error- for _, entry := range pending {- if s.isTrackedInternal(entry.ID) {- continue- }- switch entry.Metadata["action"] {- case "create_volume_backup", "create_system_backup", "run_system_volume_backups", "scheduled_volume_backup", "scheduled_system_backup":- errMessage := message- _, err := s.CompleteActivity(ctx, entry.ID, activity.StatusFailed, message, &errMessage)- result = errors.Join(result, err)- }- }- return result+ func(sweeps.Candidate) (activity.Status, string) {+ return activity.StatusFailed, "Backup interrupted by Arcane restart"+ },+ )+ return err}// SyncJobRun updates a durable job summary without taking an execution slot.// Only the queue owns its lifecycle, including reopening it for an explicit retry.func (s *ActivityService) SyncJobRun(ctx context.Context, run scheduler.Run, name string) error {- if err := s.checkInitInternal(); err != nil {+ if err := s.checkInit(); err != nil {return err}if run.ActivityID == "" {return errors.New("job activity ID is required")}- lock := s.publishLockInternal(run.ActivityID)+ lock := s.publishLock(run.ActivityID)lock.Lock()defer lock.Unlock()- model := jobActivityInternal(run, name)+ model := jobActivity(run, name)changed := falseif err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {var existing Activity@@ -1696,15 +1199,13 @@ func (s *ActivityService) SyncJobRun(ctx context.Context, run scheduler.Run, namif changed {// Ordinary activities cannot reopen. Job summaries can, after an explicit// retry; the publish lock and durable timestamp reject stale snapshots.- s.terminalPublishedMu.Lock()- delete(s.terminalPublished, model.ID)- s.terminalPublishedMu.Unlock()- s.publishActivityInternal(activityToDTOInternal(&model))+ s.pubsub.Reopen(model.ID)+ s.pubsub.PublishActivity(activityToDTO(&model))}return nil}-func jobActivityInternal(run scheduler.Run, name string) Activity {+func jobActivity(run scheduler.Run, name string) Activity {status := activity.StatusQueuedswitch run.Status {case scheduler.Running:@@ -1718,43 +1219,52 @@ func jobActivityInternal(run scheduler.Run, name string) Activity {case scheduler.Queued, scheduler.Waiting, scheduler.Retrying:}message := cmp.Or(run.Outcome.Message, name+": "+string(run.Status))- model := Activity{- ID: run.ActivityID, CreatedAt: run.CreatedAt, UpdatedAt: new(run.UpdatedAt),- EnvironmentID: run.EnvironmentID, BatchID: new(run.ID), Type: activity.TypeJobRun, Status: status,- ResourceType: new("job"), ResourceID: new(run.JobID), ResourceName: new(name),- StartedAt: run.CreatedAt, Step: string(run.Status), LatestMessage: message,- Metadata: database.JSON{- "jobId": run.JobID, "runId": run.ID, "environmentId": run.EnvironmentID,- "jobStatus": string(run.Status), "trigger": run.Trigger, "attemptCount": run.AttemptCount,- "runUpdatedAt": run.UpdatedAt.Format(time.RFC3339Nano), "domainActivityId": run.Outcome.ActivityID,- },- }- if run.RequestedBy != "" {- model.StartedByUserID = new(run.RequestedBy)- model.StartedByUsername = new(run.RequestedBy)- }- if run.Resolution != nil {- model.Metadata["resolution"] = run.Resolution- if run.Resolution.ResolvedBy != user.SystemUser.Username {- model.LatestMessage = "Resolved after review"- }- }- if isTerminalActivityStatusInternal(status) {- model.EndedAt = new(run.UpdatedAt)- model.DurationMs = new(max(int64(0), run.UpdatedAt.Sub(run.CreatedAt).Milliseconds()))- }- if status == activity.StatusFailed {- model.Error = new(message)+ terminal := slices.Contains(terminalActivityStatuses, status)+ resolvedByUser := run.Resolution != nil && run.Resolution.ResolvedBy != user.SystemUser.Username+ return Activity{+ ID: run.ActivityID,+ CreatedAt: run.CreatedAt,+ UpdatedAt: new(run.UpdatedAt),+ EnvironmentID: run.EnvironmentID,+ BatchID: new(run.ID),+ Type: activity.TypeJobRun,+ Status: status,+ ResourceType: new("job"),+ ResourceID: new(run.JobID),+ ResourceName: new(name),+ StartedByUserID: mo.EmptyableToOption(run.RequestedBy).ToPointer(),+ StartedByUsername: mo.EmptyableToOption(run.RequestedBy).ToPointer(),+ StartedAt: run.CreatedAt,+ Step: string(run.Status),+ LatestMessage: kit.Ternary(resolvedByUser, "Resolved after review", message),+ EndedAt: kit.Ternary(terminal, new(run.UpdatedAt), nil),+ DurationMs: kit.Ternary(terminal, new(max(int64(0), run.UpdatedAt.Sub(run.CreatedAt).Milliseconds())), nil),+ Error: kit.Ternary(status == activity.StatusFailed, new(message), nil),+ Metadata: func() database.JSON {+ metadata := database.JSON{+ "jobId": run.JobID,+ "runId": run.ID,+ "environmentId": run.EnvironmentID,+ "jobStatus": string(run.Status),+ "trigger": run.Trigger,+ "attemptCount": run.AttemptCount,+ "runUpdatedAt": run.UpdatedAt.Format(time.RFC3339Nano),+ "domainActivityId": run.Outcome.ActivityID,+ }+ if run.Resolution != nil {+ metadata["resolution"] = run.Resolution+ }+ return metadata+ }(),}- return model}// remoteActivityPageCeiling mirrors the agent's page clamp in PaginateAndSortDB.const remoteActivityPageCeiling = 100-// normalizeRemoteActivityParamsInternal applies the agent's paging bounds so both+// normalizeRemoteActivityParams applies the agent's paging bounds so both// sources are cut the same way; -1 stays the explicit "show all" sentinel.-func normalizeRemoteActivityParamsInternal(params pagination.QueryParams) pagination.QueryParams {+func normalizeRemoteActivityParams(params pagination.QueryParams) pagination.QueryParams {params.Start = max(params.Start, 0)switch {case params.Limit == -1:@@ -1766,9 +1276,9 @@ func normalizeRemoteActivityParamsInternal(params pagination.QueryParams) paginareturn params}-// collectActivityWindowInternal reads window rows from a source in pages the+// collectActivityWindow reads window rows from a source in pages the// agent accepts, stopping early when the source runs out. -1 reads everything.-func collectActivityWindowInternal(window int, fetchPage func(start, limit int) ([]activity.Activity, int64, error)) ([]activity.Activity, int64, error) {+func collectActivityWindow(window int, fetchPage func(start, limit int) ([]activity.Activity, int64, error)) ([]activity.Activity, int64, error) {if window == -1 {return fetchPage(0, -1)}@@ -1792,22 +1302,16 @@ func collectActivityWindowInternal(window int, fetchPage func(start, limit int)// ListRemoteActivities combines manager-owned summaries with agent-owned work// before slicing the page. The remote records must already match the filters// and be the agent's newest window; remoteTotal is the agent's filtered count.-func (- s *ActivityService,-) ListRemoteActivities(+func (s *ActivityService) ListRemoteActivities(ctx context.Context,environmentID string,remote []activity.Activity,remoteTotal int64,params pagination.QueryParams,-) (- []activity.Activity,- pagination.Response,- error,-) {- params = normalizeRemoteActivityParamsInternal(params)+) ([]activity.Activity, pagination.Response, error) {+ params = normalizeRemoteActivityParams(params)window := kit.Ternary(params.Limit == -1, -1, params.Start+params.Limit)- activities, localTotal, err := collectActivityWindowInternal(window, func(start, limit int) ([]activity.Activity, int64, error) {+ activities, localTotal, err := collectActivityWindow(window, func(start, limit int) ([]activity.Activity, int64, error) {local := paramslocal.Start, local.Limit = start, limitrows, page, err := s.ListActivitiesPaginated(ctx, environmentID, local)@@ -1825,7 +1329,7 @@ func (}total := remoteTotal + localTotalconfig := pagination.Config[activity.Activity]{SortBindings: []pagination.SortBinding[activity.Activity]{{- Fn: func(a, b activity.Activity) int { return compareActivitiesInternal(a, b, params) },+ Fn: func(a, b activity.Activity) int { return compareActivities(a, b, params) },}}}pageParams := paramspageParams.Order = pagination.SortAsc@@ -1833,7 +1337,7 @@ func (return activities, pagination.BuildResponse(total, 0, params), nil}-func compareActivitiesInternal(a, b activity.Activity, params pagination.QueryParams) int {+func compareActivities(a, b activity.Activity, params pagination.QueryParams) int {if params.Sort == "" {aActive := a.Status == activity.StatusQueued || a.Status == activity.StatusRunningbActive := b.Status == activity.StatusQueued || b.Status == activity.StatusRunningdiff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.goindex 3fa42d4e3..3c0a44e9f 100644--- a/backend/internal/activity/service_test.go+++ b/backend/internal/activity/service_test.go@@ -31,7 +31,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils")-func setupActivityServiceTestDBInternal(t *testing.T) *database.DB {+func setupActivityServiceTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})@@ -40,9 +40,9 @@ func setupActivityServiceTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func TestActivityServiceLifecycleInternal(t *testing.T) {+func TestActivityServiceLifecycle(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)progress := 5@@ -106,14 +106,14 @@ func TestActivityServiceLifecycleInternal(t *testing.T) {require.Equal(t, "Pull complete", detail.Messages[1].Message)}-func TestActivityServiceStreamFanoutInternal(t *testing.T) {+func TestActivityServiceStreamFanout(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)- events, _, unsubscribe := service.Subscribe("0")+ events, _, unsubscribe := service.pubsub.Subscribe("0")defer unsubscribe()- otherEvents, _, unsubscribeOther := service.Subscribe("0")+ otherEvents, _, unsubscribeOther := service.pubsub.Subscribe("0")defer unsubscribeOther()created, err := service.StartActivity(ctx, StartActivityRequest{@@ -123,11 +123,11 @@ func TestActivityServiceStreamFanoutInternal(t *testing.T) {})require.NoError(t, err)- first := receiveActivityEventInternal(t, events)+ first := receiveActivityEvent(t, events)require.Equal(t, "activity", first.Type)require.Equal(t, created.ID, first.ActivityID)require.NotNil(t, first.Activity)- other := receiveActivityEventInternal(t, otherEvents)+ other := receiveActivityEvent(t, otherEvents)require.Equal(t, first, other)require.NotSame(t, first.Activity, other.Activity)@@ -138,7 +138,7 @@ func TestActivityServiceStreamFanoutInternal(t *testing.T) {})require.NoError(t, err)- messageEvent := receiveActivityEventInternal(t, events)+ messageEvent := receiveActivityEvent(t, events)require.Equal(t, "message", messageEvent.Type)require.Equal(t, created.ID, messageEvent.ActivityID)require.Equal(t, activity.TypeProjectDeploy, messageEvent.ActivityType)@@ -146,12 +146,12 @@ func TestActivityServiceStreamFanoutInternal(t *testing.T) {require.Equal(t, "Deploying services", messageEvent.Message.Message)require.Nil(t, messageEvent.Activity)require.Nil(t, messageEvent.Activities)- require.Equal(t, messageEvent, receiveActivityEventInternal(t, otherEvents))+ require.Equal(t, messageEvent, receiveActivityEvent(t, otherEvents))}-func TestActivityServiceRetentionCleanupInternal(t *testing.T) {+func TestActivityServiceRetentionCleanup(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)created, err := service.StartActivity(ctx, StartActivityRequest{@@ -183,9 +183,9 @@ func TestActivityServiceRetentionCleanupInternal(t *testing.T) {require.Zero(t, messageCount)}-func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanupInternal(t *testing.T) {+func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanup(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)created, err := service.StartActivity(ctx, StartActivityRequest{@@ -209,26 +209,9 @@ func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanupInternal(t *trequire.EqualValues(t, 1, activityCount)}-func TestActivityServiceSubscribeMarksMissedEventsWhenBufferFullInternal(t *testing.T) {- service := NewActivityService(nil, nil)-- events, missedEvents, unsubscribe := service.Subscribe("0")- defer unsubscribe()-- // Message-type events are not coalesced; overflowing the delivery channel- // plus the bounded pending FIFO must flag missed events so the stream- // handler resends a snapshot.- total := cap(events) + subscriberMessageQueueLimit + 50- for range total {- service.publishInternal("0", activity.StreamEvent{Type: "message"})- }- require.True(t, missedEvents())- require.False(t, missedEvents())-}--func TestActivityServiceDeleteHistoryPreservesActiveActivitiesInternal(t *testing.T) {+func TestActivityServiceDeleteHistoryPreservesActiveActivities(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)completed, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})@@ -256,9 +239,9 @@ func TestActivityServiceDeleteHistoryPreservesActiveActivitiesInternal(t *testinrequire.ElementsMatch(t, []string{running.ID, remoteCompleted.ID}, []string{remaining[0].ID, remaining[1].ID})}-func TestActivityServicePruneHistoryByAgeAndCountInternal(t *testing.T) {+func TestActivityServicePruneHistoryByAgeAndCount(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)oldActivity, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})@@ -292,7 +275,7 @@ func TestActivityServicePruneHistoryByAgeAndCountInternal(t *testing.T) {var terminalRemoteCount int64require.NoError(t, db.Model(&Activity{}).- Where("environment_id = ? AND status IN ?", "remote-1", terminalActivityStatusesInternal()).+ Where("environment_id = ? AND status IN ?", "remote-1", terminalActivityStatuses).Count(&terminalRemoteCount).Error)require.EqualValues(t, 2, terminalRemoteCount)@@ -305,47 +288,9 @@ func TestActivityServicePruneHistoryByAgeAndCountInternal(t *testing.T) {require.Zero(t, oldCount)}-func TestActivitySubscriberCoalescesProgressEventsInternal(t *testing.T) {- db := setupActivityServiceTestDBInternal(t)- service := NewActivityService(db, nil)-- events, missedEvents, unsubscribe := service.Subscribe("0")- defer unsubscribe()-- const updates = 500- for i := 1; i <= updates; i++ {- progress := i * 100 / updates- service.publishActivityInternal(activity.Activity{- ID: "act-1",- EnvironmentID: "0",- Status: activity.StatusRunning,- Progress: &progress,- })- }-- received := 0- deadline := time.After(5 * time.Second)- for {- select {- case event := <-events:- received++- require.Equal(t, "activity", event.Type)- if event.Activity != nil && event.Activity.Progress != nil && *event.Activity.Progress == 100 {- // The consumer was idle during publishing, so the backlog must- // have been coalesced instead of delivered event-for-event.- require.Less(t, received, updates)- require.False(t, missedEvents())- return- }- case <-deadline:- require.FailNow(t, "did not receive final coalesced progress event")- }- }-}--func TestActivityServiceListOrderStableUnderProgressUpdatesInternal(t *testing.T) {+func TestActivityServiceListOrderStableUnderProgressUpdates(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)older, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull})@@ -381,7 +326,7 @@ func TestActivityServiceListOrderStableUnderProgressUpdatesInternal(t *testing.Trequire.Equal(t, expected, listIDs())}-func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, context.Context) {+func setupQueuedActivityService(t *testing.T) (*ActivityService, *settings.SettingsService, context.Context) {t.Helper()ctx := t.Context()@@ -396,13 +341,13 @@ func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, contextrequire.NoError(t, err)sqlDB.SetMaxOpenConns(1)- localSettings, err := newSettingsServiceForTestInternal(t, ctx, wrapped)+ localSettings, err := newSettingsServiceForTest(t, ctx, wrapped)require.NoError(t, err)- require.NoError(t, localSettings.SetIntSetting(ctx, maxConcurrentActivitiesSettingKey, 1))- return NewActivityService(wrapped, localSettings), ctx+ require.NoError(t, localSettings.SetIntSetting(ctx, "maxConcurrentActivities", 1))+ return NewActivityService(wrapped, localSettings), localSettings, ctx}-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()svc, err := settings.NewSettingsService(ctx, db)if err == nil {@@ -411,8 +356,8 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *dareturn svc, err}-func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFreesInternal(t *testing.T) {- service, ctx := setupQueuedActivityServiceInternal(t)+func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFrees(t *testing.T) {+ service, _, ctx := setupQueuedActivityService(t)first, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})require.NoError(t, err)@@ -459,8 +404,8 @@ func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFreesInternal(t *tesrequire.NoError(t, service.AwaitActivitySlot(ctx, secondDeferred.ID, "0"))}-func TestActivityServiceLimitIncreaseKeepsCountingActiveSlotsInternal(t *testing.T) {- service, ctx := setupQueuedActivityServiceInternal(t)+func TestActivityServiceLimitIncreaseKeepsCountingActiveSlots(t *testing.T) {+ service, localSettings, ctx := setupQueuedActivityService(t)running, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})require.NoError(t, err)@@ -474,12 +419,12 @@ func TestActivityServiceLimitIncreaseKeepsCountingActiveSlotsInternal(t *testing// Raising the limit to 2 must admit the waiter while still counting the// original holder, leaving no capacity for a third activity.- require.NoError(t, service.limiter.settings.SetIntSetting(ctx, maxConcurrentActivitiesSettingKey, 2))+ require.NoError(t, localSettings.SetIntSetting(ctx, "maxConcurrentActivities", 2))select {case awaitErr := <-awaitDone:require.NoError(t, awaitErr)- case <-time.After(2 * slotWaitRecheckInterval):+ case <-time.After(4 * time.Second):require.FailNow(t, "waiter was not admitted after the limit increase")}@@ -488,8 +433,8 @@ func TestActivityServiceLimitIncreaseKeepsCountingActiveSlotsInternal(t *testingrequire.Equal(t, "queued", string(third.Status))}-func TestActivityServiceCancelWhileQueuedUnblocksAwaitInternal(t *testing.T) {- service, ctx := setupQueuedActivityServiceInternal(t)+func TestActivityServiceCancelWhileQueuedUnblocksAwait(t *testing.T) {+ service, _, ctx := setupQueuedActivityService(t)_, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})require.NoError(t, err)@@ -511,14 +456,14 @@ func TestActivityServiceCancelWhileQueuedUnblocksAwaitInternal(t *testing.T) {}}-func TestActivityServiceCompleteActivityRejectsUninitializedServiceInternal(t *testing.T) {+func TestActivityServiceCompleteActivityRejectsUninitializedService(t *testing.T) {service := NewActivityService(nil, nil)_, err := service.CompleteActivity(t.Context(), "any-id", activity.StatusSuccess, "done", nil)require.Error(t, err)}-func TestActivityServiceTrackAndRequestCancelInternal(t *testing.T) {- db := setupActivityServiceTestDBInternal(t)+func TestActivityServiceTrackAndRequestCancel(t *testing.T) {+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)// Mirror the handler flow: work runs under an app-lifecycle runtime context.@@ -552,9 +497,9 @@ func TestActivityServiceTrackAndRequestCancelInternal(t *testing.T) {require.False(t, service.RequestCancel(created.ID))}-func TestActivityServiceCancelActivityInternal(t *testing.T) {+func TestActivityServiceCancelActivity(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)// An untracked running activity (e.g. after a restart) is finalized directly.@@ -579,9 +524,9 @@ func TestActivityServiceCancelActivityInternal(t *testing.T) {require.ErrorIs(t, err, gorm.ErrRecordNotFound)}-func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {+func TestActivityServiceFailStaleImageUpdateChecks(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)staleCheck, err := service.StartActivity(ctx, StartActivityRequest{@@ -647,9 +592,9 @@ func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {require.Equal(t, activity.StatusSuccess, completed.Status)}-func TestActivityServiceFailAbandonedActivitiesInternal(t *testing.T) {+func TestActivityServiceFailAbandonedActivities(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)abandoned, err := service.StartActivity(ctx, StartActivityRequest{@@ -696,9 +641,9 @@ func TestActivityServiceFailAbandonedActivitiesInternal(t *testing.T) {require.Equal(t, activity.StatusRunning, freshRow.Status)}-func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {+func TestActivityServiceResolveStaleAutoUpdateActivities(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)selfUpdateRun, err := service.StartActivity(ctx, StartActivityRequest{@@ -744,7 +689,7 @@ func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {require.Equal(t, activity.StatusRunning, other.Status)}-func receiveActivityEventInternal(t *testing.T, events <-chan activity.StreamEvent) activity.StreamEvent {+func receiveActivityEvent(t *testing.T, events <-chan activity.StreamEvent) activity.StreamEvent {t.Helper()select {@@ -756,12 +701,12 @@ func receiveActivityEventInternal(t *testing.T, events <-chan activity.StreamEve}}-// TestActivityServiceAppendMessagesBatchInternal verifies a batch lands all+// TestActivityServiceAppendMessagesBatch verifies a batch lands all// messages in order and coalesces the activity update to the batch's last// message, last progress, and last step.-func TestActivityServiceAppendMessagesBatchInternal(t *testing.T) {+func TestActivityServiceAppendMessagesBatch(t *testing.T) {ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)created, err := service.StartActivity(ctx, StartActivityRequest{@@ -803,48 +748,8 @@ func TestActivityServiceAppendMessagesBatchInternal(t *testing.T) {require.ErrorContains(t, err, "activity not found")}-// TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal verifies-// that a non-terminal activity snapshot published after the activity's-// terminal event — a goroutine that committed before CompleteActivity but-// publishes after it — is dropped instead of reverting subscribers to a-// running state no later event would correct.-func TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal(t *testing.T) {- ctx := t.Context()- db := setupActivityServiceTestDBInternal(t)- service := NewActivityService(db, nil)-- created, err := service.StartActivity(ctx, StartActivityRequest{- EnvironmentID: "0",- Type: activity.TypeImagePull,- LatestMessage: "Pull queued",- })- require.NoError(t, err)- stale := *created // snapshot taken while the activity is still active-- events, _, unsubscribe := service.Subscribe("0")- defer unsubscribe()-- completed, err := service.CompleteActivity(ctx, created.ID, activity.StatusSuccess, "", nil)- require.NoError(t, err)- require.Equal(t, activity.StatusSuccess, completed.Status)-- event := receiveActivityEventInternal(t, events)- require.Equal(t, "activity", event.Type)- require.Equal(t, activity.StatusSuccess, event.Activity.Status)-- require.False(t, service.admitActivityPublishInternal(stale))- require.True(t, service.admitActivityPublishInternal(*completed))-- service.publishActivityInternal(stale)- select {- case localEvent := <-events:- t.Fatalf("stale non-terminal snapshot reached subscriber: %+v", localEvent)- case <-time.After(100 * time.Millisecond):- }-}-func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)now := time.Now().UTC()run := scheduler.Run{ID: "run", ActivityID: "summary", JobID: "auto-update", EnvironmentID: "0", Status: scheduler.NeedsAttention, CreatedAt: now.Add(-time.Hour), UpdatedAt: now}@@ -859,9 +764,7 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {require.Equal(t, activity.StatusQueued, detail.Activity.Status)require.Nil(t, detail.Activity.EndedAt)require.Nil(t, detail.Activity.Error)- require.Empty(t, service.slotReleases)require.Empty(t, service.running)- require.NotContains(t, service.terminalPublished, run.ActivityID)count, err := service.ResolveOrphanedQueuedActivities(t.Context())require.NoError(t, err)require.Zero(t, count)@@ -878,7 +781,7 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {require.Equal(t, "remote", legacy.EnvironmentID)// Startup reconciliation also repairs legacy rows without replaying the job.require.NoError(t, db.Model(&legacy).Update("environment_id", "0").Error)- events, _, unsubscribe := service.Subscribe("remote")+ events, _, unsubscribe := service.pubsub.Subscribe("remote")defer unsubscribe()require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))select {@@ -919,7 +822,7 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {}func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)for _, environmentID := range []string{"0", "private"} {require.NoError(@@ -954,7 +857,7 @@ func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {require.NoError(t, err)require.EqualValues(t, 2, response.TotalItems)require.Len(t, activities, 1)- require.True(t, canReadJobActivityInternal(ctx, activities[0]))+ require.True(t, canReadJobActivity(ctx, activities[0]))seen[activities[0].ID] = true}require.Len(t, seen, 2)@@ -965,7 +868,7 @@ func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {require.Equal(t, "private-pull", activities[0].ID)require.False(t,- canReadJobActivityInternal(+ canReadJobActivity(ctx,activity.Activity{Type: activity.TypeJobRun,@@ -974,13 +877,13 @@ func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {},),)- require.True(t, canReadJobActivityInternal(ctx, activity.Activity{Type: activity.TypeJobRun, Metadata: map[string]any{"environmentId": "allowed"}}))+ require.True(t, canReadJobActivity(ctx, activity.Activity{Type: activity.TypeJobRun, Metadata: map[string]any{"environmentId": "allowed"}}))privileged := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.SudoPermissionSet())activities, response, err = service.ListActivitiesPaginated(privileged, "private", pagination.QueryParams{Limit: 10})require.NoError(t, err)require.Len(t, activities, 2)require.EqualValues(t, 2, response.TotalItems)- require.False(t, canReadJobActivityInternal(privileged, activity.Activity{Type: activity.TypeJobRun}))+ require.False(t, canReadJobActivity(privileged, activity.Activity{Type: activity.TypeJobRun}))permissions.PerEnv["0"][authz.PermActivitiesDelete] = struct{}{}deleted, err := service.DeleteHistory(ctx, "private")require.NoError(t, err)@@ -1009,7 +912,7 @@ func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)require.NoError(t, db.AutoMigrate(&environment.Environment{}, ®istry.ContainerRegistry{}, &gitrepo.GitRepository{}, &s3.S3Destination{}))sqlDB, err := db.DB.DB()require.NoError(t, err)@@ -1066,7 +969,7 @@ func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {}func TestActivityService_StartActivityWithIDIsIdempotent(t *testing.T) {- db := setupActivityServiceTestDBInternal(t)+ db := setupActivityServiceTestDB(t)service := NewActivityService(db, nil)first, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "first"})diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.goindex 9fd7d45f3..830492928 100644--- a/backend/internal/backup/service.go+++ b/backend/internal/backup/service.go@@ -520,6 +520,10 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Cliencommand = append(command, listedID)}}+ if len(command) == 2 && len(requested) > 0 {+ slog.DebugContext(ctx, "No requested snapshots found in repository; skipping forget", "repositoryId", repository.ID, "requested", requested)+ return nil+ }if len(command) == 2 {command = []string{"prune"}}diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.goindex b2f7e2b7e..e91cb85e4 100644--- a/backend/internal/bootstrap/bootstrap.go+++ b/backend/internal/bootstrap/bootstrap.go@@ -16,6 +16,7 @@ import ("github.com/labstack/echo/v5""github.com/moby/moby/client""github.com/subosito/gotenv"+ kit "go.getarcane.app/kit/pkg""go.getarcane.app/sys/crypto""go.opentelemetry.io/contrib/bridges/otelslog""go.uber.org/fx"@@ -23,6 +24,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/di""github.com/getarcaneapp/arcane/backend/v2/internal/docker"@@ -144,7 +146,12 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *databasefx.Supply(cfg, db, cancelApp),fx.Provide(func() context.Context { return appCtx },- newConfiguredHTTPClient,+ func(cfg *config.Config) *http.Client {+ return httpx.NewHTTPClient(httpxtypes.ClientOptions{+ Timeout: kit.Ternary(cfg.HTTPClientTimeout > 0, time.Duration(cfg.HTTPClientTimeout)*time.Second, 10*time.Second),+ TLSHandshakeTimeout: kit.Ternary(cfg.HTTPClientTimeout > 0, 10*time.Second, 5*time.Second),+ })+ },),di.ActorOptions,di.ServiceOptions,@@ -162,7 +169,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *databasereturn logger}),fx.StartTimeout(5*time.Minute),- fx.StopTimeout(30*time.Second),+ fx.StopTimeout(8*time.Second),)}@@ -179,18 +186,6 @@ func isWeakProductionEncryptionKey(encryptionKey, localEnvironment string, agentreturn len(strings.TrimPrefix(key, "raw:")) < 32}-func newConfiguredHTTPClient(cfg *config.Config) *http.Client {- options := httpxtypes.ClientOptions{- Timeout: 10 * time.Second,- TLSHandshakeTimeout: 5 * time.Second,- }- if cfg.HTTPClientTimeout > 0 {- options.Timeout = time.Duration(cfg.HTTPClientTimeout) * time.Second- options.TLSHandshakeTimeout = 10 * time.Second- }- return httpx.NewHTTPClient(options)-}-type initializeStartupStateParams struct {fx.In@@ -203,6 +198,7 @@ type initializeStartupStateParams struct {Environment *environment.EnvironmentServiceGitOpsSync *gitops.GitOpsSyncServiceProject *project.ProjectService+ Container *container.ContainerServiceVariable *variable.VariableServiceDocker *docker.DockerClientServiceSwarm *swarm.SwarmService@@ -247,34 +243,30 @@ func initializeStartupState(p initializeStartupStateParams) {})startup.InitializeDefaultSettings(appCtx, runtimeCfg, p.Settings)- if err := p.Settings.NormalizeProjectsDirectory(appCtx, cfg.ProjectsDirectory); err != nil {- slog.WarnContext(appCtx, "Failed to normalize projects directory", "error", err)- }-- if err := p.Settings.NormalizeBuildsDirectory(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to normalize builds directory", "error", err)- }-- if err := p.Environment.EnsureLocalEnvironment(appCtx, cfg.AppUrl); err != nil {- slog.WarnContext(appCtx, "Failed to ensure local environment", "error", err)- }- if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)- }- if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)- }- if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)- }- if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)+ // Startup repairs are best-effort: log each failure and continue.+ for _, task := range []struct {+ name string+ run func(context.Context) error+ }{+ {name: "normalize projects directory", run: func(ctx context.Context) error {+ return p.Settings.NormalizeProjectsDirectory(ctx, cfg.ProjectsDirectory)+ }},+ {name: "normalize builds directory", run: p.Settings.NormalizeBuildsDirectory},+ {name: "ensure local environment", run: func(ctx context.Context) error { return p.Environment.EnsureLocalEnvironment(ctx, cfg.AppUrl) }},+ {name: "clean up orphaned GitOps syncs", run: p.GitOpsSync.CleanupOrphanedSyncsOnStartup},+ {name: "clean up leaked GitOps scratch directories", run: p.GitOpsSync.CleanupLeakedScratchDirsOnStartup},+ {name: "clean up leaked git clone directories", run: p.GitOpsSync.CleanupLeakedCloneDirsOnStartup},+ {name: "reconcile directory GitOps projects", run: p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup},+ {name: "recover interrupted project renames", run: p.Project.RecoverProjectRenameJournals},+ {name: "recover interrupted container redeploys", run: p.Container.RecoverInterruptedRedeploys},+ {name: "recover stuck project statuses", run: p.Project.RecoverStuckProjectStatuses},+ } {+ if err := task.run(appCtx); err != nil {+ slog.WarnContext(appCtx, "Startup task failed", "task", task.name, "error", err)+ }}p.Vulnerability.ImportLegacyReportFiles(appCtx)p.Vulnerability.BackfillScanItems(appCtx)- if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {- slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)- }if !cfg.AgentMode {if err := p.Environment.ReconcileEdgeStatusesOnStartup(appCtx); err != nil {@@ -291,19 +283,17 @@ func initializeStartupState(p initializeStartupStateParams) {}startup.TestDockerConnection(appCtx, func(ctx context.Context) error {- dockerClient, err := p.Docker.GetClient(ctx)- if err != nil {- return err- }-- version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})- if err != nil {- return err- }+ _, err := p.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (struct{}, error) {+ version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})+ if err != nil {+ return struct{}{}, err+ }- effectiveAPIVersion := cmp.Or(strings.TrimSpace(dockerClient.ClientVersion()), strings.TrimSpace(version.APIVersion))- slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)- return nil+ effectiveAPIVersion := cmp.Or(strings.TrimSpace(dockerClient.ClientVersion()), strings.TrimSpace(version.APIVersion))+ slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)+ return struct{}{}, nil+ })+ return err})if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)@@ -313,7 +303,7 @@ func initializeStartupState(p initializeStartupStateParams) {p.Role.EnsureBuiltInRoles,func(ctx context.Context) error {// Backfill legacy users.roles first so CreateDefaultAdmin's zero-global-admin gate sees them;- // runRoleStartupTasks repeats it for agent mode and retries.+ // the role startup tasks below repeat it for agent mode and retries.if err := p.Role.BackfillLegacyRoleAssignments(ctx); err != nil {slog.WarnContext(ctx, "Failed to backfill legacy role assignments before admin bootstrap", "error", err)}@@ -329,7 +319,24 @@ func initializeStartupState(p initializeStartupStateParams) {)startup.CleanupUnknownSettings(appCtx, p.Settings)- runRoleStartupTasks(appCtx, p.DB, p.Role, p.ApiKey, cfg)+ if err := p.Role.EnsureBuiltInRoles(appCtx); err != nil {+ slog.ErrorContext(appCtx, "Failed to reconcile built-in roles", "error", err)+ }+ // Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.+ if err := p.Role.BackfillLegacyRoleAssignments(appCtx); err != nil {+ slog.ErrorContext(appCtx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)+ }+ if err := p.ApiKey.BackfillApiKeyPermissions(appCtx); err != nil {+ slog.WarnContext(appCtx, "Failed to backfill API key permissions", "error", err)+ }+ if err := p.Role.ReconcileEnvOidcMappings(appCtx, cfg.OidcRoleMappings); err != nil {+ slog.ErrorContext(appCtx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)+ }+ if !cfg.AgentMode {+ if err := p.Role.AssertGlobalAdminExists(appCtx, p.DB.WithContext(appCtx)); err != nil {+ slog.ErrorContext(appCtx, "RBAC global admin guard failed", "error", err)+ }+ }// Only edge agents auto-pair; direct agents are passive and the manager dials them.if cfg.AgentMode && cfg.EdgeAgent && cfg.AgentToken != "" && cfg.ManagerApiUrl != "" {@@ -341,33 +348,6 @@ func initializeStartupState(p initializeStartupStateParams) {}}-func runRoleStartupTasks(ctx context.Context, db *database.DB, roleService *role.RoleService, apiKeyService *apikey.ApiKeyService, cfg *config.Config) {- if roleService == nil {- return- }- if err := roleService.EnsureBuiltInRoles(ctx); err != nil {- slog.ErrorContext(ctx, "Failed to reconcile built-in roles", "error", err)- }- // Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.- if err := roleService.BackfillLegacyRoleAssignments(ctx); err != nil {- slog.ErrorContext(ctx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)- }- if apiKeyService != nil {- if err := apiKeyService.BackfillApiKeyPermissions(ctx); err != nil {- slog.WarnContext(ctx, "Failed to backfill API key permissions", "error", err)- }- }- if err := roleService.ReconcileEnvOidcMappings(ctx, cfg.OidcRoleMappings); err != nil {- slog.ErrorContext(ctx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)- }- if cfg.AgentMode {- return- }- if err := roleService.AssertGlobalAdminExists(ctx, db.WithContext(ctx)); err != nil {- slog.ErrorContext(ctx, "RBAC global admin guard failed", "error", err)- }-}-func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, router *echo.Echo, _ *http.Server) {var stop func(context.Context) errorlc.Append(fx.Hook{diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 48e3257d9..ad820fd05 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -28,6 +28,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/gitops"+ "github.com/getarcaneapp/arcane/backend/v2/internal/health""github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/project"@@ -200,7 +201,8 @@ func TestHTTP2APIResponsesDoNotUseAPIGzip(t *testing.T) {Project: project.New(&project.ProjectService{}, nil),Container: container.New(&container.ContainerService{}, nil, nil, nil),Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),- System: system.New(&system.SystemService{}, nil, nil, nil),+ System: system.New(&system.SystemService{}, nil, nil),+ Health: health.New(&health.Service{}),},AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),TunnelRegistry: edge.NewTunnelRegistry(),diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.goindex 993dda67a..e214c21d3 100644--- a/backend/internal/bootstrap/router_bootstrap.go+++ b/backend/internal/bootstrap/router_bootstrap.go@@ -295,7 +295,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {}// Remaining echo handlers (WebSocket/streaming)- ws.NewWebSocketHandler(apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)+ ws.NewWebSocketHandler(ctx, apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)// Managers accept agent connections on the edge tunnel endpoint.var tunnelServer *edge.TunnelServerdiff --git a/backend/internal/bootstrap/router_bootstrap_test.go b/backend/internal/bootstrap/router_bootstrap_test.goindex 3893f99ec..d8e0049cf 100644--- a/backend/internal/bootstrap/router_bootstrap_test.go+++ b/backend/internal/bootstrap/router_bootstrap_test.go@@ -19,6 +19,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/container"+ "github.com/getarcaneapp/arcane/backend/v2/internal/health""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/swarm"@@ -40,7 +41,8 @@ func TestNewRouter_DecodesPathParams(t *testing.T) {Project: project.New(&project.ProjectService{}, nil),Container: container.New(&container.ContainerService{}, nil, nil, nil),Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),- System: system.New(&system.SystemService{}, nil, nil, nil),+ System: system.New(&system.SystemService{}, nil, nil),+ Health: health.New(&health.Service{}),},AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),TunnelRegistry: edge.NewTunnelRegistry(),@@ -156,7 +158,8 @@ func TestProxiedEnvironmentRoutesAreEdgeRoutable(t *testing.T) {Project: project.New(&project.ProjectService{}, nil),Container: container.New(&container.ContainerService{}, nil, nil, nil),Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),- System: system.New(&system.SystemService{}, nil, nil, nil),+ System: system.New(&system.SystemService{}, nil, nil),+ Health: health.New(&health.Service{}),},AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),TunnelRegistry: edge.NewTunnelRegistry(),diff --git a/backend/internal/config/config.go b/backend/internal/config/config.goindex d487ac683..3810e4a41 100644--- a/backend/internal/config/config.go+++ b/backend/internal/config/config.go@@ -35,7 +35,7 @@ type Config struct {BuildablesConfigAppUrl string `env:"APP_URL" default:"http://localhost:3552"`- DatabaseURL string `env:"DATABASE_URL" default:"file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate" options:"file"`+ DatabaseURL string `env:"DATABASE_URL" default:"file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate" options:"file"`AllowDowngrade bool `env:"ALLOW_DOWNGRADE" default:"false"`AllowCLIPasswordReset bool `env:"ALLOW_CLI_PASSWORD_RESET" default:"false"`AllowCLIMFAReset bool `env:"ALLOW_CLI_MFA_RESET" default:"false"`@@ -118,7 +118,7 @@ func Load() *Config {cfg := &Config{}loadFromEnv(cfg)applyOptions(cfg)- applyAgentModeDefaults(cfg)+ cfg.AgentMode = cfg.AgentMode || cfg.EdgeAgentapplyProxyDefaults(cfg)applyWorkspaceDefaults(cfg)@@ -172,12 +172,6 @@ func applyWorkspaceDefaults(cfg *Config) {}}-func applyAgentModeDefaults(cfg *Config) {- if cfg.EdgeAgent {- cfg.AgentMode = true- }-}-// applyProxyDefaults auto-trusts loopback proxies when LISTEN binds to a// loopback address. This is the canonical convenience-script setup// (LISTEN=127.0.0.1 + local reverse proxy like Caddy/Nginx/Traefik); auto-@@ -216,7 +210,7 @@ func loadFromEnv(cfg *Config) {envValue := os.Getenv(envTag)var fromFile bool- if hasOptionInternal(fieldType, "file") {+ if hasOption(fieldType, "file") {envValue, _, fromFile = utils.LookupEnvOrFile(envTag)}if !fromFile {@@ -226,12 +220,12 @@ func loadFromEnv(cfg *Config) {}}- setFieldValueInternal(field, fieldType, envValue)+ setFieldValue(field, fieldType, envValue)})}-// hasOptionInternal reports whether the field's `options` tag lists option.-func hasOptionInternal(fieldType reflect.StructField, option string) bool {+// hasOption reports whether the field's `options` tag lists option.+func hasOption(fieldType reflect.StructField, option string) bool {return slices.Contains(strings.Split(fieldType.Tag.Get("options"), ","), option)}@@ -241,7 +235,7 @@ func (c *Config) DeprecatedEnvVarsSet() []string {var envNames []stringv := reflect.ValueOf(c).Elem()visitConfigFields(v, func(field reflect.Value, fieldType reflect.StructField) {- if !hasOptionInternal(fieldType, "deprecated") {+ if !hasOption(fieldType, "deprecated") {return}if field.Kind() == reflect.String && field.String() == "" {@@ -315,8 +309,8 @@ func visitConfigFields(v reflect.Value, fn func(reflect.Value, reflect.StructFie}}-// setFieldValueInternal sets a reflect.Value from a string based on the field's type.-func setFieldValueInternal(field reflect.Value, fieldType reflect.StructField, value string) {+// setFieldValue sets a reflect.Value from a string based on the field's type.+func setFieldValue(field reflect.Value, fieldType reflect.StructField, value string) {ctx := context.Background() //nolint:forbidigo // Configuration loads before any request context exists.if !field.CanSet() {return@@ -350,47 +344,26 @@ func setFieldValueInternal(field reflect.Value, fieldType reflect.StructField, v}if field.Type() == reflect.TypeFor[time.Duration]() {- applyDurationDefault := func(reason string) {- envTag := fieldType.Tag.Get("env")- defaultValue := fieldType.Tag.Get("default")-- if fallback, fallbackErr := time.ParseDuration(defaultValue); fallbackErr == nil {- slog.WarnContext(ctx, "Invalid duration for config field, using tagged default",- "reason", reason,- "field", envTag,- "value", value,- "default", defaultValue)- field.SetInt(int64(fallback))- } else {- slog.WarnContext(ctx, "Invalid duration for config field and invalid tagged default",- "reason", reason,- "field", envTag,- "value", value,- "default", defaultValue)- }+ d, err := time.ParseDuration(value)+ if err == nil && d > 0 {+ field.SetInt(int64(d))+ return}-- if d, err := time.ParseDuration(value); err == nil {- if d > 0 {- field.SetInt(int64(d))- } else {- applyDurationDefault("Non-positive duration for config field")- }- } else {- applyDurationDefault("Invalid duration for config field")+ reason := kit.Ternary(err == nil, "Non-positive duration for config field", "Invalid duration for config field")+ envTag, defaultValue := fieldType.Tag.Get("env"), fieldType.Tag.Get("default")+ fallback, fallbackErr := time.ParseDuration(defaultValue)+ if fallbackErr != nil {+ slog.WarnContext(ctx, "Invalid duration for config field and invalid tagged default", "reason", reason, "field", envTag, "value", value, "default", defaultValue)+ return}+ slog.WarnContext(ctx, "Invalid duration for config field, using tagged default", "reason", reason, "field", envTag, "value", value, "default", defaultValue)+ field.SetInt(int64(fallback))return}- // Handle custom types based on underlying kind+ // String-based types like AppEnvironment; os.FileMode is handled as Uint32 above.if field.Type().ConvertibleTo(reflect.TypeFor[string]()) {- // String-based types like AppEnvironmentfield.Set(reflect.ValueOf(value).Convert(field.Type()))- } else if field.Type() == reflect.TypeFor[os.FileMode]() {- // os.FileMode- if i, err := strconv.ParseUint(value, 8, 32); err == nil {- field.Set(reflect.ValueOf(os.FileMode(i)))- }}}@@ -463,10 +436,7 @@ func (c *Config) MaskSensitive() map[string]any {envTag := cmp.Or(fieldType.Tag.Get("env"), fieldType.Name)// Fields with "file" option are considered sensitive- optionsTag := fieldType.Tag.Get("options")- isSensitive := strings.Contains(optionsTag, "file")-- if isSensitive {+ if hasOption(fieldType, "file") {// Mask sensitive valuesstrVal := fmt.Sprintf("%v", field.Interface())result[envTag] = kit.Ternary(strVal != "", "****", "(empty)")diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.goindex 2c200ca28..74a3146d6 100644--- a/backend/internal/config/config_test.go+++ b/backend/internal/config/config_test.go@@ -557,7 +557,7 @@ func TestConfig_JWTRefreshExpiry(t *testing.T) {})}-func TestSetFieldValueInternal_DurationUsesFieldTagDefault(t *testing.T) {+func TestSetFieldValue_DurationUsesFieldTagDefault(t *testing.T) {type durationConfig struct {SessionTimeout time.Duration `env:"SESSION_TIMEOUT" default:"2h"`}@@ -568,7 +568,7 @@ func TestSetFieldValueInternal_DurationUsesFieldTagDefault(t *testing.T) {fieldType, ok := v.Type().FieldByName("SessionTimeout")require.True(t, ok)- setFieldValueInternal(field, fieldType, "invalid-duration")+ setFieldValue(field, fieldType, "invalid-duration")assert.Equal(t, 2*time.Hour, cfg.SessionTimeout)}diff --git a/backend/internal/container/model.go b/backend/internal/container/model.gonew file mode 100644index 000000000..a6680696f--- /dev/null+++ b/backend/internal/container/model.go@@ -0,0 +1,15 @@+package container++import "time"++// redeployJournalKeyPrefix keys redeploy journals in the KV store by the original container ID.+const redeployJournalKeyPrefix = "container_redeploy_journal:"++// RedeployJournal records a container Arcane renamed to a backup name, so recovery only touches its own backups.+type RedeployJournal struct {+ OriginalID string `json:"originalId"`+ OriginalName string `json:"originalName"`+ BackupName string `json:"backupName"`+ NewName string `json:"newName"`+ StartedAt time.Time `json:"startedAt"`+}diff --git a/backend/internal/container/service.go b/backend/internal/container/service.goindex 2e0093769..e2e245c89 100644--- a/backend/internal/container/service.go+++ b/backend/internal/container/service.go@@ -3,6 +3,7 @@ package containerimport ("cmp""context"+ "encoding/json/v2""errors""fmt""io"@@ -42,12 +43,14 @@ import (dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/image"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/iconcatalog"@@ -57,6 +60,12 @@ const (containerGroupByProject = "project"containerNoProjectGroup = "No Project"containerIconMetadataTTL = 5 * time.Second+ // leaseWaitTimeout bounds how long a container operation waits for one already running on the same container.+ leaseWaitTimeout = 30 * time.Second+ // rollbackTimeout bounds a redeploy rollback, which runs detached from the request so it completes after cancellation.+ rollbackTimeout = 2 * timeouts.DefaultDockerAPI+ // redeployBackupInfix joins a container's name and a Unix timestamp to name the original kept during a redeploy.+ redeployBackupInfix = "-arcane-redeploy-")type ContainerService struct {@@ -65,8 +74,11 @@ type ContainerService struct {imageService *image.ImageServicesettingsService *settings.SettingsServiceprojectService *project.ProjectService+ kvService *kv.KVServiceiconMetaCache *hot.HotCache[string, projects.ArcaneComposeMetadata]stats *stats.Service+ // opLock serializes mutating operations on the same container.+ opLock utils.KeyedMutex}type ContainerListResult struct {@@ -81,7 +93,7 @@ type containerLifecycleAction struct {eventType event.EventTypemetadata database.JSONwarnOnLogError bool- runContainerAction func(*client.Client) error+ runContainerAction func(context.Context) error}type ExecSession struct {@@ -105,6 +117,7 @@ func NewContainerService(imageService *image.ImageService,settingsService *settings.SettingsService,projectService *project.ProjectService,+ kvService *kv.KVService,) *ContainerService {return &ContainerService{eventService: eventService,@@ -112,6 +125,7 @@ func NewContainerService(imageService: imageService,settingsService: settingsService,projectService: projectService,+ kvService: kvService,iconMetaCache: hot.NewHotCache[string, projects.ArcaneComposeMetadata](hot.LRU, 1024).WithTTL(containerIconMetadataTTL).WithJanitor().@@ -162,125 +176,133 @@ func buildCleanNetworkingConfig(containerInspect container.InspectResponse, apiVfunc (s *ContainerService) pullRedeployImage(ctx context.Context,- dockerClient *client.Client,imageName, containerID, containerName, action string,credentials []containerregistry.Credential,user usertypes.Actor,) error {- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))- defer pullCancel()-- pullOptions, authErr := s.imageService.PullOptionsWithAuth(ctx, imageName, credentials)- if authErr != nil {- slog.WarnContext(- ctx, "failed to get registry authentication for container recreate pull; proceeding without auth",- "image", imageName,- "error", authErr.Error(),- )- pullOptions = client.ImagePullOptions{}- }-- defer s.eventService.BeginDockerResourceSuppressionWindow("image", "", imageName)()- reader, pullErr := dockerClient.ImagePull(pullCtx, imageName, pullOptions)- if pullErr != nil && image.ShouldRetryAnonymousPull(pullOptions, pullErr) {- slog.WarnContext(- ctx, "container recreate image pull failed with registry auth; retrying anonymously",- "image", imageName,- "error", pullErr.Error(),- )- pullOptions = client.ImagePullOptions{}- reader, pullErr = dockerClient.ImagePull(pullCtx, imageName, pullOptions)- }+ progressWriter, _ := ctx.Value(types.ProgressWriterKey{}).(io.Writer)+ pullErr := s.imageService.Pull(ctx, image.PullRequest{+ Ref: imageName,+ Progress: progressWriter,+ User: user,+ Credentials: credentials,+ })if pullErr != nil {- if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", pullErr, database.JSON{- "action": action,- "step": "pull_image_timeout",- "image": imageName,- })- return fmt.Errorf("image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", imageName)- }-s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", pullErr, database.JSON{"action": action,- "step": "pull_image",+ "step": kit.Ternary(errors.Is(pullErr, context.DeadlineExceeded), "pull_image_timeout", "pull_image"),"image": imageName,})- return fmt.Errorf("failed to pull image %s: %w", imageName, pullErr)}- defer func() { _ = reader.Close() }()-- progressWriter, _ := ctx.Value(types.ProgressWriterKey{}).(io.Writer)- logWriter := docker.NewLogLineWriter(progressWriter)- defer func() { _ = logWriter.Close() }()+ return pullErr+}- streamErr := docker.RenderJSONMessageStream(reader, logWriter)- if streamErr != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", streamErr, database.JSON{- "action": action,- "step": "complete_pull",- "image": imageName,- })- return fmt.Errorf("failed to complete image pull: %w", streamErr)+// RecoverInterruptedRedeploys resolves journaled redeploy backups a restart left behind: a backup whose replacement+// is running is removed, and a backup with no replacement gets its original name back but stays stopped.+func (s *ContainerService) RecoverInterruptedRedeploys(ctx context.Context) error {+ entries, err := s.kvService.ListByPrefix(ctx, redeployJournalKeyPrefix)+ if err != nil {+ return fmt.Errorf("list redeploy journals: %w", err)+ }+ if len(entries) == 0 {+ return nil}- return nil-}+ list, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })+ if err != nil {+ return fmt.Errorf("list containers for redeploy recovery: %w", err)+ }-func (s *ContainerService) prepareContainerForRedeploy(- ctx context.Context,- dockerClient *client.Client,- containerID, containerName, backupName string,- wasRunning bool,- user usertypes.Actor,-) error {- s.eventService.MarkDockerExpectation("container", containerID, containerName)- if containerName != "" {- if _, err := dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: backupName}); err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", err, database.JSON{- "action": "redeploy",- "step": "rename_old",- "backupName": backupName,- })- return fmt.Errorf("failed to rename existing container: %w", err)+ byName := make(map[string]container.Summary, len(list.Items))+ for _, c := range list.Items {+ for _, name := range c.Names {+ byName[strings.TrimPrefix(name, "/")] = c}}- if !wasRunning {- return nil- }+ var firstErr error+ var removed, restored, skipped int+ for _, entry := range entries {+ var journal RedeployJournal+ if unmarshalErr := json.Unmarshal([]byte(entry.Value), &journal); unmarshalErr != nil {+ slog.WarnContext(ctx, "Failed to decode redeploy journal", "key", entry.Key, "error", unmarshalErr)+ continue+ }- _, err := dockerClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: new(30)})- if err == nil {- return nil- }+ backup, backupExists := byName[journal.BackupName]+ if !backupExists {+ s.clearRedeployJournal(ctx, journal.OriginalID)+ continue+ }+ if backup.ID != journal.OriginalID {+ slog.WarnContext(ctx, "Skipping redeploy backup recovery: container is not the one Arcane renamed", "container", journal.BackupName, "id", backup.ID, "journaledId", journal.OriginalID)+ s.clearRedeployJournal(ctx, journal.OriginalID)+ skipped+++ continue+ }- if containerName != "" {- if _, renameErr := dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: containerName}); renameErr != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", renameErr, database.JSON{- "action": "redeploy",- "step": "restore_name_after_stop_failure",- })+ replacement, exists := byName[cmp.Or(journal.NewName, journal.OriginalName)]+ switch {+ case exists && replacement.State == container.StateRunning:+ if _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, backup.ID, client.ContainerRemoveOptions{Force: true})+ }); removeErr != nil {+ slog.WarnContext(ctx, "Failed to remove leftover redeploy backup", "container", journal.BackupName, "error", removeErr)+ firstErr = cmp.Or(firstErr, fmt.Errorf("remove redeploy backup %s: %w", journal.BackupName, removeErr))+ continue+ }+ s.clearRedeployJournal(ctx, journal.OriginalID)+ removed+++ case exists:+ slog.WarnContext(ctx, "Leaving redeploy backup in place: replacement exists but is not running",+ "container", journal.BackupName,+ "replacement", cmp.Or(journal.NewName, journal.OriginalName),+ "state", replacement.State)+ skipped+++ default:+ if _, renameErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRenameResult, error) {+ return dockerClient.ContainerRename(ctx, backup.ID, client.ContainerRenameOptions{NewName: journal.OriginalName})+ }); renameErr != nil {+ slog.WarnContext(ctx, "Failed to restore leftover redeploy backup", "container", journal.BackupName, "error", renameErr)+ firstErr = cmp.Or(firstErr, fmt.Errorf("restore redeploy backup %s: %w", journal.BackupName, renameErr))+ continue+ }+ s.clearRedeployJournal(ctx, journal.OriginalID)+ byName[journal.OriginalName] = backup+ restored+++ slog.WarnContext(ctx, "Restored original container from interrupted redeploy; start it manually if it should be running", "container", journal.OriginalName, "state", backup.State)}}- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", err, database.JSON{- "action": "redeploy",- "step": "stop",- })- return fmt.Errorf("failed to stop container: %w", err)+ if removed+restored+skipped > 0 {+ slog.InfoContext(ctx, "Recovered interrupted container redeploys", "removed", removed, "restored", restored, "skipped", skipped)+ }+ return firstErr+}++// clearRedeployJournal drops the journal once its backup is resolved; a leftover entry is retried at startup.+func (s *ContainerService) clearRedeployJournal(ctx context.Context, originalID string) {+ if err := s.kvService.Delete(ctx, redeployJournalKeyPrefix+originalID); err != nil {+ slog.WarnContext(ctx, "Failed to clear redeploy journal", "containerId", originalID, "error", err)+ }}func (s *ContainerService) restoreContainerAfterRedeployFailure(ctx context.Context,- dockerClient *client.Client,containerID, containerName, backupName, failedStep string,wasRunning bool,user usertypes.Actor,) {+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rollbackTimeout)+ defer cancel()+s.eventService.MarkDockerExpectation("container", containerID, containerName)if wasRunning {- if _, startErr := dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); startErr != nil {+ if _, startErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{})+ }); startErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", startErr, database.JSON{"action": "redeploy","step": "restore_start_original",@@ -293,21 +315,77 @@ func (s *ContainerService) restoreContainerAfterRedeployFailure(return}- if _, renameErr := dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: containerName}); renameErr != nil {+ if _, renameErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRenameResult, error) {+ return dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: containerName})+ }); renameErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, backupName, user.ID, user.Username, "0", renameErr, database.JSON{"action": "redeploy","step": "restore_name","failedStep": failedStep,})+ return}+ s.clearRedeployJournal(ctx, containerID)}-func (s *ContainerService) runContainerLifecycleAction(ctx context.Context, containerID string, user usertypes.Actor, cfg containerLifecycleAction) error {+// renameToRedeployBackup journals the original before renaming it to its backup name, so startup+// recovery only acts on backups Arcane created.+func (s *ContainerService) renameToRedeployBackup(ctx context.Context, containerID, containerName, backupName, newName string, user usertypes.Actor) error {+ journal, err := json.Marshal(RedeployJournal{+ OriginalID: containerID,+ OriginalName: containerName,+ BackupName: backupName,+ NewName: newName,+ StartedAt: time.Now().UTC(),+ })+ if err != nil {+ return fmt.Errorf("marshal redeploy journal: %w", err)+ }+ if setErr := s.kvService.Set(ctx, redeployJournalKeyPrefix+containerID, string(journal)); setErr != nil {+ return fmt.Errorf("write redeploy journal: %w", setErr)+ }++ if _, renameErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRenameResult, error) {+ return dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: backupName})+ }); renameErr != nil {+ s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", renameErr, database.JSON{+ "action": "redeploy",+ "step": "rename_old",+ "backupName": backupName,+ })+ s.clearRedeployJournal(context.WithoutCancel(ctx), containerID)+ return fmt.Errorf("failed to rename existing container: %w", renameErr)+ }+ return nil+}++// stopContainer waits out a 30s grace period, so it runs without the Docker API deadline.+func (s *ContainerService) stopContainer(ctx context.Context, containerID string) error {dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{"action": cfg.action})return fmt.Errorf("failed to connect to Docker: %w", err)}+ _, err = dockerClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: new(30)})+ return err+}++// lockContainer takes the container's operation lease, waiting up to leaseWaitTimeout for an operation already running on it.+func (s *ContainerService) lockContainer(ctx context.Context, op, containerID string) (func(), error) {+ waitCtx, cancel := context.WithTimeout(ctx, leaseWaitTimeout)+ defer cancel()+ unlock, err := s.opLock.LockContext(waitCtx, "0:container:"+containerID)+ if err != nil {+ return nil, fmt.Errorf("%s %s: waiting for in-progress operation: %w", op, containerID, err)+ }+ return unlock, nil+}++func (s *ContainerService) runContainerLifecycleAction(ctx context.Context, containerID string, user usertypes.Actor, cfg containerLifecycleAction) error {+ unlock, err := s.lockContainer(ctx, cfg.action+" container", containerID)+ if err != nil {+ return err+ }+ defer unlock()metadata := database.JSON{"action": cfg.action,@@ -324,7 +402,7 @@ func (s *ContainerService) runContainerLifecycleAction(ctx context.Context, cont}defer s.eventService.BeginDockerResourceSuppressionWindow("container", containerID, "")()- err = cfg.runContainerAction(dockerClient)+ err = cfg.runContainerAction(ctx)if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{"action": cfg.action})}@@ -336,8 +414,10 @@ func (s *ContainerService) StartContainer(ctx context.Context, containerID strinaction: "start",eventType: event.EventTypeContainerStart,warnOnLogError: true,- runContainerAction: func(dockerClient *client.Client) error {- _, err := dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{})+ runContainerAction: func(ctx context.Context) error {+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{})+ })return err},})@@ -347,9 +427,8 @@ func (s *ContainerService) StopContainer(ctx context.Context, containerID stringreturn s.runContainerLifecycleAction(ctx, containerID, user, containerLifecycleAction{action: "stop",eventType: event.EventTypeContainerStop,- runContainerAction: func(dockerClient *client.Client) error {- _, err := dockerClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: new(30)})- return err+ runContainerAction: func(ctx context.Context) error {+ return s.stopContainer(ctx, containerID)},})}@@ -358,8 +437,12 @@ func (s *ContainerService) RestartContainer(ctx context.Context, containerID strreturn s.runContainerLifecycleAction(ctx, containerID, user, containerLifecycleAction{action: "restart",eventType: event.EventTypeContainerRestart,- runContainerAction: func(dockerClient *client.Client) error {- _, err := dockerClient.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})+ runContainerAction: func(ctx context.Context) error {+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return fmt.Errorf("failed to connect to Docker: %w", err)+ }+ _, err = dockerClient.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})return err},})@@ -373,8 +456,10 @@ func (s *ContainerService) KillContainer(ctx context.Context, containerID, signaeventType: event.EventTypeContainerKill,metadata: database.JSON{"signal": signal},warnOnLogError: true,- runContainerAction: func(dockerClient *client.Client) error {- _, err := dockerClient.ContainerKill(ctx, containerID, client.ContainerKillOptions{Signal: signal})+ runContainerAction: func(ctx context.Context) error {+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerKillResult, error) {+ return dockerClient.ContainerKill(ctx, containerID, client.ContainerKillOptions{Signal: signal})+ })return err},})@@ -386,8 +471,10 @@ func (s *ContainerService) PauseContainer(ctx context.Context, containerID strinaction: "pause",eventType: event.EventTypeContainerPause,warnOnLogError: true,- runContainerAction: func(dockerClient *client.Client) error {- _, err := dockerClient.ContainerPause(ctx, containerID, client.ContainerPauseOptions{})+ runContainerAction: func(ctx context.Context) error {+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerPauseResult, error) {+ return dockerClient.ContainerPause(ctx, containerID, client.ContainerPauseOptions{})+ })return err},})@@ -399,8 +486,10 @@ func (s *ContainerService) UnpauseContainer(ctx context.Context, containerID straction: "unpause",eventType: event.EventTypeContainerUnpause,warnOnLogError: true,- runContainerAction: func(dockerClient *client.Client) error {- _, err := dockerClient.ContainerUnpause(ctx, containerID, client.ContainerUnpauseOptions{})+ runContainerAction: func(ctx context.Context) error {+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerUnpauseResult, error) {+ return dockerClient.ContainerUnpause(ctx, containerID, client.ContainerUnpauseOptions{})+ })return err},})@@ -451,16 +540,15 @@ func (s *ContainerService) CommitContainer(ctx context.Context, containerID stri}func (s *ContainerService) RedeployContainer(ctx context.Context, containerID string, user usertypes.Actor) (string, error) {- dockerClient, err := s.dockerService.GetClient(ctx)+ unlock, err := s.lockContainer(ctx, "redeploy container", containerID)if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{- "action": "redeploy",- "step": "get_client",- })- return "", fmt.Errorf("failed to connect to Docker: %w", err)+ return "", err}+ defer unlock()- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ })if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{"action": "redeploy",@@ -480,7 +568,9 @@ func (s *ContainerService) RedeployContainer(ctx context.Context, containerID st}containerName := strings.TrimPrefix(containerInfo.Name, "/")- apiVersion := compat.DetectDockerAPIVersion(ctx, dockerClient)+ apiVersion, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {+ return compat.DetectDockerAPIVersion(ctx, dockerClient), nil+ })currentContainerID, currentContainerErr := cgroup.CurrentContainerID()if labels.ShouldDisableArcaneServerRedeploy(containerInfo.Config.Labels, containerInfo.ID, currentContainerID, currentContainerErr) {@@ -550,14 +640,13 @@ func (s *ContainerService) RedeployContainer(ctx context.Context, containerID st}if containerInfo.Config.Image != "" {- if pullRedeployImageErr := s.pullRedeployImage(ctx, dockerClient, containerInfo.Config.Image, containerID, containerName, "redeploy", nil, user); pullRedeployImageErr != nil {+ if pullRedeployImageErr := s.pullRedeployImage(ctx, containerInfo.Config.Image, containerID, containerName, "redeploy", nil, user); pullRedeployImageErr != nil {return "", pullRedeployImageErr}}return s.recreateContainer(ctx,- dockerClient,containerInfo,"redeploy",containerName,@@ -574,7 +663,6 @@ func (s *ContainerService) RedeployContainer(ctx context.Context, containerID st// rename-as-backup so the original can be restored when create or start fails. Shared by redeploy and edit.func (s *ContainerService) recreateContainer(ctx context.Context,- dockerClient *client.Client,containerInfo container.InspectResponse,action, newName string,newConfig *container.Config,@@ -583,10 +671,7 @@ func (s *ContainerService) recreateContainer(apiVersion string,eventType event.EventType,user usertypes.Actor,-) (- string,- error,-) {+) (string, error) {containerName := strings.TrimPrefix(containerInfo.Name, "/")wasRunning := kit.FromPtr(containerInfo.State).Running@@ -595,28 +680,25 @@ func (s *ContainerService) recreateContainer(stopAfterCreate := wasRunning && kit.FromPtr(containerInfo.HostConfig).AutoRemovedefer s.eventService.BeginDockerResourceSuppressionWindow("container", containerInfo.ID, containerName)()- backupName := fmt.Sprintf("%s-arcane-redeploy-%d", cmp.Or(containerName, func() string {- if len(containerInfo.ID) < 12 {- return "arcane-redeploy"- }- return "arcane-redeploy-" + containerInfo.ID[:12]- }()), time.Now().Unix())+ backupName := fmt.Sprintf("%s%s%d", cmp.Or(containerName, "arcane-redeploy-"+containerInfo.ID[:min(12, len(containerInfo.ID))]), redeployBackupInfix, time.Now().Unix())// Stopping the original made the daemon delete it, so rebuild it from the inspected config.// A returned error means the workload is down and is joined into the start error.- restoreAutoRemoveOriginal := func() error {+ restoreAutoRemoveOriginal := func(ctx context.Context) error {originalConfig := *containerInfo.Configif len(containerInfo.ID) >= 12 && originalConfig.Hostname == containerInfo.ID[:12] {originalConfig.Hostname = "" // keep: copying Docker's 24-field container.Config to clear one field is unreadable}defer s.eventService.BeginDockerResourceSuppressionWindow("container", "", containerName)()- restoreResp, restoreCreateErr := compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{- Config: &originalConfig,- HostConfig: containerInfo.HostConfig,- NetworkingConfig: buildCleanNetworkingConfig(containerInfo, apiVersion),- Name: containerName,- }, apiVersion)+ restoreResp, restoreCreateErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{+ Config: &originalConfig,+ HostConfig: containerInfo.HostConfig,+ NetworkingConfig: buildCleanNetworkingConfig(containerInfo, apiVersion),+ Name: containerName,+ }, apiVersion)+ })if restoreCreateErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", restoreCreateErr, database.JSON{"action": "redeploy",@@ -626,7 +708,9 @@ func (s *ContainerService) recreateContainer(}defer s.eventService.BeginDockerResourceSuppressionWindow("container", restoreResp.ID, containerName)()- if _, restoreStartErr := dockerClient.ContainerStart(ctx, restoreResp.ID, client.ContainerStartOptions{}); restoreStartErr != nil {+ if _, restoreStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, restoreResp.ID, client.ContainerStartOptions{})+ }); restoreStartErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", restoreResp.ID, containerName, user.ID, user.Username, "0", restoreStartErr, database.JSON{"action": "redeploy","step": "restore_start_original",@@ -643,8 +727,21 @@ func (s *ContainerService) recreateContainer(return nil}- if err := s.prepareContainerForRedeploy(ctx, dockerClient, containerInfo.ID, containerName, backupName, wasRunning && !stopAfterCreate, user); err != nil {- return "", err+ s.eventService.MarkDockerExpectation("container", containerInfo.ID, containerName)+ if containerName != "" {+ if err := s.renameToRedeployBackup(ctx, containerInfo.ID, containerName, backupName, newName, user); err != nil {+ return "", err+ }+ }+ if wasRunning && !stopAfterCreate {+ if err := s.stopContainer(ctx, containerInfo.ID); err != nil {+ s.restoreContainerAfterRedeployFailure(ctx, containerInfo.ID, containerName, backupName, "stop", false, user)+ s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", err, database.JSON{+ "action": "redeploy",+ "step": "stop",+ })+ return "", fmt.Errorf("failed to stop container: %w", err)+ }}if len(containerInfo.ID) >= 12 && newConfig.Hostname == containerInfo.ID[:12] {@@ -652,14 +749,16 @@ func (s *ContainerService) recreateContainer(}defer s.eventService.BeginDockerResourceSuppressionWindow("container", "", newName)()- createResp, err := compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{- Config: newConfig,- HostConfig: newHostConfig,- NetworkingConfig: networkingConfig,- Name: newName,- }, apiVersion)+ createResp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{+ Config: newConfig,+ HostConfig: newHostConfig,+ NetworkingConfig: networkingConfig,+ Name: newName,+ }, apiVersion)+ })if err != nil {- s.restoreContainerAfterRedeployFailure(ctx, dockerClient, containerInfo.ID, containerName, backupName, "create", wasRunning && !stopAfterCreate, user)+ s.restoreContainerAfterRedeployFailure(ctx, containerInfo.ID, containerName, backupName, "create", wasRunning && !stopAfterCreate, user)s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", err, database.JSON{"action": action,"step": "create",@@ -671,14 +770,16 @@ func (s *ContainerService) recreateContainer(defer s.eventService.BeginDockerResourceSuppressionWindow("container", createResp.ID, newName)()if stopAfterCreate {- if _, containerStopErr := dockerClient.ContainerStop(ctx, containerInfo.ID, client.ContainerStopOptions{Timeout: new(30)}); containerStopErr != nil {- if _, removeErr := dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true}); removeErr != nil {+ if containerStopErr := s.stopContainer(ctx, containerInfo.ID); containerStopErr != nil {+ if _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true})+ }); removeErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", createResp.ID, containerName, user.ID, user.Username, "0", removeErr, database.JSON{"action": action,"step": "cleanup_failed_stop",})}- s.restoreContainerAfterRedeployFailure(ctx, dockerClient, containerInfo.ID, containerName, backupName, "stop", false, user)+ s.restoreContainerAfterRedeployFailure(ctx, containerInfo.ID, containerName, backupName, "stop", false, user)s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", containerStopErr, database.JSON{"action": action,"step": "stop",@@ -689,11 +790,17 @@ func (s *ContainerService) recreateContainer(restartPolicies := []container.RestartPolicyMode{container.RestartPolicyAlways, container.RestartPolicyUnlessStopped, container.RestartPolicyOnFailure}if wasRunning || slices.Contains(restartPolicies, kit.FromPtr(containerInfo.HostConfig).RestartPolicy.Name) {- if _, err = dockerClient.ContainerStart(ctx, createResp.ID, client.ContainerStartOptions{}); err != nil {+ if _, err = s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, createResp.ID, client.ContainerStartOptions{})+ }); err != nil {// Remove the replacement and roll back to the original. A deferred-stop auto-remove original// is rebuilt from its inspected config; a failed restore joins the start error.+ rollbackCtx, cancelRollback := context.WithTimeout(context.WithoutCancel(ctx), rollbackTimeout)+ defer cancelRollback()s.eventService.MarkDockerExpectation("container", createResp.ID, containerName)- _, removeErr := dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true})+ _, removeErr := s.dockerService.Call(rollbackCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true})+ })if removeErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", createResp.ID, containerName, user.ID, user.Username, "0", removeErr, database.JSON{"action": action,@@ -701,11 +808,12 @@ func (s *ContainerService) recreateContainer(})}if stopAfterCreate {- if restoreErr := restoreAutoRemoveOriginal(); restoreErr != nil {+ if restoreErr := restoreAutoRemoveOriginal(rollbackCtx); restoreErr != nil {err = errors.Join(err, kit.Ternary(removeErr != nil, fmt.Errorf("failed to remove replacement container: %w", removeErr), nil), restoreErr)}+ s.clearRedeployJournal(rollbackCtx, containerInfo.ID)} else {- s.restoreContainerAfterRedeployFailure(ctx, dockerClient, containerInfo.ID, containerName, backupName, "start", wasRunning, user)+ s.restoreContainerAfterRedeployFailure(rollbackCtx, containerInfo.ID, containerName, backupName, "start", wasRunning, user)}s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", createResp.ID, containerName, user.ID, user.Username, "0", err, database.JSON{"action": action,@@ -725,20 +833,25 @@ func (s *ContainerService) recreateContainer("image", newConfig.Image,)- // After a deferred stop the daemon has already auto-removed the original.- if !stopAfterCreate {- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerInfo.ID, client.ContainerRemoveOptions{+ // After a deferred stop the daemon has already auto-removed the original. A failed removal+ // keeps the journal so startup recovery removes the backup.+ if stopAfterCreate {+ s.clearRedeployJournal(ctx, containerInfo.ID)+ } else if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, containerInfo.ID, client.ContainerRemoveOptions{Force: true,RemoveVolumes: false,RemoveLinks: false,- }); containerRemoveErr != nil {- slog.WarnContext(- ctx, "failed to remove old container after successful recreate",- "containerId", containerInfo.ID,- "backupName", backupName,- "error", containerRemoveErr,- )- }+ })+ }); containerRemoveErr != nil {+ slog.WarnContext(+ ctx, "failed to remove old container after successful recreate",+ "containerId", containerInfo.ID,+ "backupName", backupName,+ "error", containerRemoveErr,+ )+ } else {+ s.clearRedeployJournal(ctx, containerInfo.ID)}if logErr := s.eventService.LogContainerEvent(ctx, eventType, createResp.ID, newName, user.ID, user.Username, "0", database.JSON{@@ -960,12 +1073,9 @@ func buildEditNetworkingConfig(containerInspect container.InspectResponse, req *// GetContainerEditConfig returns the purpose-built DTO backing the container// edit form.func (s *ContainerService) GetContainerEditConfig(ctx context.Context, containerID string) (containertypes.EditConfig, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return containertypes.EditConfig{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ })if err != nil {return containertypes.EditConfig{}, fmt.Errorf("failed to inspect container: %w", err)}@@ -982,16 +1092,15 @@ func (s *ContainerService) GetContainerEditConfig(ctx context.Context, container// inspected config and recreates it. Sections not owned by the edit form are// preserved from the existing container.func (s *ContainerService) EditContainer(ctx context.Context, containerID string, req containertypes.Edit, user usertypes.Actor) (string, error) {- dockerClient, err := s.dockerService.GetClient(ctx)+ unlock, err := s.lockContainer(ctx, "edit container", containerID)if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{- "action": "edit",- "step": "get_client",- })- return "", fmt.Errorf("failed to connect to Docker: %w", err)+ return "", err}+ defer unlock()- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ })if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{"action": "edit",@@ -1023,12 +1132,16 @@ func (s *ContainerService) EditContainer(ctx context.Context, containerID stringnewName := cmp.Or(strings.TrimPrefix(strings.TrimSpace(kit.FromPtr(req.Name)), "/"), containerName)if newName != containerName {- if _, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, newName, client.ContainerInspectOptions{}); inspectErr == nil {+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, newName, client.ContainerInspectOptions{})+ }); inspectErr == nil {return "", fmt.Errorf("container name %s: %w", newName, common.ErrContainerNameTaken)}}- apiVersion := compat.DetectDockerAPIVersion(ctx, dockerClient)+ apiVersion, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {+ return compat.DetectDockerAPIVersion(ctx, dockerClient), nil+ })newHostConfig := kit.FromPtr(containerInfo.HostConfig)newConfig, err := applyEditToContainerConfig(*containerInfo.Config, newHostConfig.PortBindings, req)@@ -1046,23 +1159,22 @@ func (s *ContainerService) EditContainer(ctx context.Context, containerID string// Unlike redeploy, edit only pulls when the effective image is missing// locally so an unrelated config change never surprise-upgrades the image.if newConfig.Image != "" {- if _, inspectErr := dockerClient.ImageInspect(ctx, newConfig.Image); inspectErr != nil {- if pullRedeployImageErr := s.pullRedeployImage(ctx, dockerClient, newConfig.Image, containerID, containerName, "edit", req.Credentials, user); pullRedeployImageErr != nil {+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, newConfig.Image)+ }); inspectErr != nil {+ if pullRedeployImageErr := s.pullRedeployImage(ctx, newConfig.Image, containerID, containerName, "edit", req.Credentials, user); pullRedeployImageErr != nil {return "", pullRedeployImageErr}}}- return s.recreateContainer(ctx, dockerClient, containerInfo, "edit", newName, &newConfig, &newHostConfig, networkingConfig, apiVersion, event.EventTypeContainerUpdate, user)+ return s.recreateContainer(ctx, containerInfo, "edit", newName, &newConfig, &newHostConfig, networkingConfig, apiVersion, event.EventTypeContainerUpdate, user)}func (s *ContainerService) GetContainerByReference(ctx context.Context, ref string) (*container.InspectResponse, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- containerInspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, ref, client.ContainerInspectOptions{})+ containerInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, ref, client.ContainerInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("container not found: %w", err)}@@ -1126,21 +1238,17 @@ var containerProcessesPsArgs = [][]string{// GetContainerProcesses returns Docker's `top` snapshot for a container.func (s *ContainerService) GetContainerProcesses(ctx context.Context, containerID string) (containertypes.Processes, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return containertypes.Processes{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- topCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer cancel()-- var result client.ContainerTopResult- for _, args := range containerProcessesPsArgs {- result, err = dockerClient.ContainerTop(topCtx, containerID, client.ContainerTopOptions{Arguments: args})- if err == nil || errdefs.IsNotFound(err) || errdefs.IsConflict(err) || topCtx.Err() != nil {- break+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerTopResult, error) {+ var result client.ContainerTopResult+ var err error+ for _, args := range containerProcessesPsArgs {+ result, err = dockerClient.ContainerTop(ctx, containerID, client.ContainerTopOptions{Arguments: args})+ if err == nil || errdefs.IsNotFound(err) || errdefs.IsConflict(err) || ctx.Err() != nil {+ break+ }}- }+ return result, err+ })if err != nil {return containertypes.Processes{}, common.Classify(func() error {switch {@@ -1176,31 +1284,18 @@ func (s *ContainerService) GetContainerNameByID(ctx context.Context, id string)}func (s *ContainerService) DeleteContainer(ctx context.Context, containerID string, force, removeVolumes bool, user usertypes.Actor) error {- dockerClient, err := s.dockerService.GetClient(ctx)+ unlock, err := s.lockContainer(ctx, "delete container", containerID)if err != nil {- s.eventService.LogErrorEvent(- ctx,- event.EventTypeContainerError,- "container",- containerID,- "",- user.ID,- user.Username,- "0",- err,- database.JSON{- "action": "delete",- "force": force,- "removeVolumes": removeVolumes,- },- )- return fmt.Errorf("failed to connect to Docker: %w", err)+ return err}+ defer unlock()// Get container mounts before deletion if we need to remove volumesvar volumesToRemove []stringif removeVolumes {- containerJSON, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ containerJSON, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ })if inspectErr == nil {for _, mount := range containerJSON.Container.Mounts {// Only collect named volumes (not bind mounts or tmpfs)@@ -1214,17 +1309,19 @@ func (s *ContainerService) DeleteContainer(ctx context.Context, containerID stridefer s.eventService.BeginDockerResourceSuppressionWindow("container", containerID, "")()volumeReleases := make([]func(), 0, len(volumesToRemove))defer func() {- for i := len(volumeReleases) - 1; i >= 0; i-- {- volumeReleases[i]()+ for _, release := range slices.Backward(volumeReleases) {+ release()}}()for _, volumeName := range volumesToRemove {volumeReleases = append(volumeReleases, s.eventService.BeginDockerResourceSuppressionWindow("volume", volumeName, volumeName))}- _, err = dockerClient.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{- Force: force,- RemoveVolumes: removeVolumes,- RemoveLinks: false,+ _, err = s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{+ Force: force,+ RemoveVolumes: removeVolumes,+ RemoveLinks: false,+ })})if err != nil {s.eventService.LogErrorEvent(@@ -1248,7 +1345,9 @@ func (s *ContainerService) DeleteContainer(ctx context.Context, containerID stri// Remove named volumes if requestedfor _, volumeName := range volumesToRemove {- if _, removeErr := dockerClient.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: false}); removeErr != nil {+ if _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeRemoveResult, error) {+ return dockerClient.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: false})+ }); removeErr != nil {// Log but don't fail if volume removal fails (might be in use by another container)s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", volumeName, "", user.ID, user.Username, "0", removeErr, database.JSON{"action": "delete", "container": containerID})}@@ -1277,25 +1376,23 @@ func (s *ContainerService) CreateContainer(*container.InspectResponse,error,) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", "", containerName, user.ID, user.Username, "0", err, database.JSON{"action": "create", "image": config.Image})- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, inspectErr := dockerClient.ImageInspect(ctx, config.Image); inspectErr != nil {+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, config.Image)+ }); inspectErr != nil {// Image not found locally, need to pull it- if pullErr := s.pullRedeployImage(ctx, dockerClient, config.Image, "", containerName, "create", credentials, user); pullErr != nil {+ if pullErr := s.pullRedeployImage(ctx, config.Image, "", containerName, "create", credentials, user); pullErr != nil {return nil, pullErr}}defer s.eventService.BeginDockerResourceSuppressionWindow("container", "", containerName)()- resp, err := compat.ContainerCreateWithCompatibility(ctx, dockerClient, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,- NetworkingConfig: networkingConfig,- Name: containerName,+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return compat.ContainerCreateWithCompatibility(ctx, dockerClient, client.ContainerCreateOptions{+ Config: config,+ HostConfig: hostConfig,+ NetworkingConfig: networkingConfig,+ Name: containerName,+ })})if err != nil {s.eventService.LogErrorEvent(@@ -1325,8 +1422,12 @@ func (s *ContainerService) CreateContainer(slog.WarnContext(ctx, "could not log container stop action", "error", logErr)}- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ })s.eventService.LogErrorEvent(ctx,event.EventTypeContainerError,@@ -1346,7 +1447,9 @@ func (s *ContainerService) CreateContainer(return nil, fmt.Errorf("failed to start container: %w", containerStartErr)}- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, resp.ID, client.ContainerInspectOptions{})+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, resp.ID, client.ContainerInspectOptions{})+ })if err != nil {s.eventService.LogErrorEvent(ctx,@@ -1985,14 +2088,16 @@ func (d *demuxedLogs) Close() error {}func (s *ContainerService) openLogs(ctx context.Context, containerID string, options client.ContainerLogsOptions) (io.ReadCloser, client.ContainerInspectResult, error) {- dockerClient, err := s.dockerService.GetClient(ctx)+ containerInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ })if err != nil {- return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to connect to Docker: %w", err)+ return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to inspect container for logs: %w", err)}- containerInspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to inspect container for logs: %w", err)+ return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to connect to Docker: %w", err)}logs, err := dockerClient.ContainerLogs(ctx, containerID, options)diff --git a/backend/internal/container/service_test.go b/backend/internal/container/service_test.goindex 5a85b2d31..e1bd2367e 100644--- a/backend/internal/container/service_test.go+++ b/backend/internal/container/service_test.go@@ -41,6 +41,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/event"imagedomain "github.com/getarcaneapp/arcane/backend/v2/internal/image""github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"@@ -144,7 +145,7 @@ func TestContainerSummaryIconsAppliedAfterGrouping(t *testing.T) {db := setupProjectTestDB(t)settingsService, err := settings.NewSettingsService(t.Context(), db)require.NoError(t, err)- service := NewContainerService(nil, nil, nil, settingsService, project.NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil))+ service := NewContainerService(nil, nil, nil, settingsService, project.NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil), nil)dockerContainers := []container.Summary{{ID: "app", Names: []string{"/app"}, Labels: map[string]string{"com.docker.compose.project": "media",@@ -323,6 +324,7 @@ func TestContainerServiceCommitContainerCallsDockerAPI(t *testing.T) {nil,nil,nil,+ nil,)out, err := svc.CommitContainer(t.Context(), "container-1", containertypes.CommitRequest{@@ -370,6 +372,7 @@ func TestContainerServiceCommitContainerOmitsReferenceWhenRepositoryEmpty(t *tesnil,nil,nil,+ nil,)out, err := svc.CommitContainer(t.Context(), "container-1", containertypes.CommitRequest{@@ -427,7 +430,7 @@ func setupProjectTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&project.Project{}, &settings.SettingVariable{}, &imageupdate.ImageUpdateRecord{}, &event.Event{}))+ require.NoError(t, db.AutoMigrate(&project.Project{}, &settings.SettingVariable{}, &imageupdate.ImageUpdateRecord{}, &event.Event{}, &kv.KVEntry{}))return &database.DB{DB: db}}@@ -629,8 +632,8 @@ func TestRecreateContainerDaemonCorrelation(t *testing.T) {w.WriteHeader(http.StatusNoContent)}))t.Cleanup(server.Close)- dockerClient := newTestDockerClient(t, server)- svc := NewContainerService(events, nil, nil, nil, nil)+ kvService := kv.NewKVService(db)+ svc := NewContainerService(events, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, nil, nil, kvService)info := container.InspectResponse{ID: "old-id",Name: "/web",@@ -640,7 +643,6 @@ func TestRecreateContainerDaemonCorrelation(t *testing.T) {}id, err := svc.recreateContainer(t.Context(),- dockerClient,info,"redeploy","web",@@ -673,12 +675,171 @@ func TestRecreateContainerDaemonCorrelation(t *testing.T) {)require.True(t, events.ShouldSuppressDaemonEvent("container", "old-id", "", ""))require.True(t, events.ShouldSuppressDaemonEvent("container", "new-id", "", ""))+ journals, err := kvService.ListByPrefix(t.Context(), redeployJournalKeyPrefix)+ require.NoError(t, err)+ require.Empty(t, journals)+}++func TestRecreateContainerRollsBackAfterRequestCancellation(t *testing.T) {+ db := setupProjectTestDB(t)+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ var mu sync.Mutex+ var observed []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ path := dockerTestPath(r.URL.Path)+ mu.Lock()+ observed = append(observed, r.Method+" "+path)+ mu.Unlock()+ switch path {+ case "/containers/create":+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusCreated)+ if err := json.NewEncoder(w).Encode(map[string]any{"Id": "new-id", "Warnings": []string{}}); err != nil {+ t.Errorf("encode container creation response: %v", err)+ }+ case "/containers/new-id/start":+ cancel()+ w.WriteHeader(http.StatusInternalServerError)+ default:+ w.WriteHeader(http.StatusNoContent)+ }+ }))+ t.Cleanup(server.Close)+ kvService := kv.NewKVService(db)+ svc := NewContainerService(event.NewEventService(db, nil, nil), docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, nil, nil, kvService)+ info := container.InspectResponse{+ ID: "old-id",+ Name: "/web",+ State: &container.State{Running: true},+ Config: &container.Config{Image: "app:latest"},+ HostConfig: &container.HostConfig{},+ }++ _, err := svc.recreateContainer(ctx, info, "redeploy", "web", info.Config, info.HostConfig, nil, "1.41", event.EventTypeContainerDeploy, user.SystemUser)+ require.Error(t, err)+ require.ErrorIs(t, ctx.Err(), context.Canceled)++ mu.Lock()+ defer mu.Unlock()+ require.Equal(+ t,+ []string{+ "POST /containers/old-id/rename",+ "POST /containers/old-id/stop",+ "POST /containers/create",+ "POST /containers/new-id/start",+ "DELETE /containers/new-id",+ "POST /containers/old-id/start",+ "POST /containers/old-id/rename",+ },+ observed,+ )+ journals, err := kvService.ListByPrefix(t.Context(), redeployJournalKeyPrefix)+ require.NoError(t, err)+ require.Empty(t, journals)+}++func TestRecoverInterruptedRedeploys(t *testing.T) {+ backupName := "web" + redeployBackupInfix + "1700000000"+ backup := container.Summary{+ ID: "backup-id",+ Names: []string{"/" + backupName},+ State: container.StateExited,+ }+ journal := RedeployJournal{+ OriginalID: "backup-id",+ OriginalName: "web",+ BackupName: backupName,+ NewName: "web",+ }+ tests := []struct {+ name string+ journals []RedeployJournal+ containers []container.Summary+ want []string+ wantJournals int+ }{+ {+ name: "replacement running removes backup",+ journals: []RedeployJournal{journal},+ containers: []container.Summary{+ backup,+ {ID: "new-id", Names: []string{"/web"}, State: container.StateRunning},+ },+ want: []string{"DELETE /containers/backup-id?force=1"},+ },+ {+ name: "missing replacement restores original name",+ journals: []RedeployJournal{journal},+ containers: []container.Summary{backup},+ want: []string{"POST /containers/backup-id/rename?name=web"},+ },+ {+ name: "stopped replacement leaves both",+ journals: []RedeployJournal{journal},+ containers: []container.Summary{+ backup,+ {ID: "new-id", Names: []string{"/web"}, State: container.StateExited},+ },+ wantJournals: 1,+ },+ {+ name: "backup with a different ID is skipped",+ journals: []RedeployJournal{journal},+ containers: []container.Summary{+ {ID: "other-id", Names: []string{"/" + backupName}, State: container.StateExited},+ },+ },+ {+ name: "name-matching container without a journal is left alone",+ containers: []container.Summary{backup},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ kvService := kv.NewKVService(setupProjectTestDB(t))+ for _, j := range tt.journals {+ payload, err := json.Marshal(j)+ require.NoError(t, err)+ require.NoError(t, kvService.Set(t.Context(), redeployJournalKeyPrefix+j.OriginalID, string(payload)))+ }+ var mu sync.Mutex+ var mutations []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ path := dockerTestPath(r.URL.Path)+ if path == "/containers/json" {+ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(tt.containers); err != nil {+ t.Errorf("encode container list: %v", err)+ }+ return+ }+ mu.Lock()+ mutations = append(mutations, r.Method+" "+path+kit.Ternary(r.URL.RawQuery != "", "?"+r.URL.RawQuery, ""))+ mu.Unlock()+ w.WriteHeader(http.StatusNoContent)+ }))+ t.Cleanup(server.Close)+ svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, nil, nil, kvService)++ require.NoError(t, svc.RecoverInterruptedRedeploys(t.Context()))++ mu.Lock()+ defer mu.Unlock()+ require.Equal(t, tt.want, mutations)+ journals, err := kvService.ListByPrefix(t.Context(), redeployJournalKeyPrefix)+ require.NoError(t, err)+ require.Len(t, journals, tt.wantJournals)+ })+ }}func TestBuildSummariesUsesContainerTagPolicyUpdates(t *testing.T) {settingsService, err := settings.NewSettingsService(t.Context(), setupProjectTestDB(t))require.NoError(t, err)- service := NewContainerService(nil, nil, nil, settingsService, nil)+ service := NewContainerService(nil, nil, nil, settingsService, nil, nil)strategyLabel := "com.getarcaneapp.arcane.updater.strategy"containers := []container.Summary{{ID: "first", Image: "app:3.1.0", ImageID: "shared-image"},@@ -790,7 +951,7 @@ func TestContainerServiceGetContainerProcesses(t *testing.T) {t.Cleanup(server.Close)settingsService, err := settings.NewSettingsService(t.Context(), setupProjectTestDB(t))require.NoError(t, err)- svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil)+ svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil, nil)processes, err := svc.GetContainerProcesses(t.Context(), "container-1")wantArgs := []string{strings.Join(containerProcessesPsArgs[0], " ")}@@ -818,7 +979,7 @@ func TestContainerServiceGetContainerProcessesPropagatesContext(t *testing.T) {t.Cleanup(server.Close)settingsService, err := settings.NewSettingsService(t.Context(), setupProjectTestDB(t))require.NoError(t, err)- svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil)+ svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil, nil)ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)defer cancel()@@ -966,6 +1127,7 @@ func newResourceSortService(t *testing.T, fixture *resourceStatsServer) *Containimagedomain.NewImageService(db, dockerService, nil, nil, nil, nil, settingsService, nil),settingsService,nil,+ nil,)}@@ -1309,6 +1471,7 @@ func newLogsTestService(t *testing.T, tty bool, logsBody []byte, gotQuery *url.Vnil,nil,nil,+ nil,)}diff --git a/backend/internal/dashboard/service.go b/backend/internal/dashboard/service.goindex 676c72e5a..fb454d799 100644--- a/backend/internal/dashboard/service.go+++ b/backend/internal/dashboard/service.go@@ -467,16 +467,16 @@ func (s *DashboardService) ObserveInventory(meter metric.Meter) error {// observeSwarm reports nodes and services only when the local engine is a swarm manager; the ping keeps non-swarm hosts cheap.func (s *DashboardService) observeSwarm(ctx context.Context, o metric.Observer, g *inventoryGauges) {- cli, err := s.dockerService.GetClient(ctx)- if err != nil {- return- }- ping, err := cli.Ping(ctx, client.PingOptions{})+ ping, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.PingResult, error) {+ return cli.Ping(ctx, client.PingOptions{})+ })if err != nil || ping.SwarmStatus == nil || !ping.SwarmStatus.ControlAvailable {return}- if nodes, nodeErr := cli.NodeList(ctx, client.NodeListOptions{}); nodeErr != nil {+ if nodes, nodeErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NodeListResult, error) {+ return cli.NodeList(ctx, client.NodeListOptions{})+ }); nodeErr != nil {slog.DebugContext(ctx, "Skipping swarm node metrics", "error", nodeErr)} else {counts := map[[2]string]int64{}@@ -488,7 +488,9 @@ func (s *DashboardService) observeSwarm(ctx context.Context, o metric.Observer,}}- services, err := cli.ServiceList(ctx, client.ServiceListOptions{Status: true})+ services, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ServiceListResult, error) {+ return cli.ServiceList(ctx, client.ServiceListOptions{Status: true})+ })if err != nil {slog.DebugContext(ctx, "Skipping swarm service metrics", "error", err)returndiff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.goindex b0489104d..03edded9c 100644--- a/backend/internal/dashboard/service_test.go+++ b/backend/internal/dashboard/service_test.go@@ -394,7 +394,7 @@ func TestDashboardService_GetSnapshot_CachesFullSnapshotsPerIconCatalog(t *testidockerSvc := newDashboardTestDockerService(t, settingsSvc, containers, images, nil)require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "autoUpdateExcludedContainers", "running-app"))- containerSvc := container.NewContainerService(nil, dockerSvc, nil, settingsSvc, nil)+ containerSvc := container.NewContainerService(nil, dockerSvc, nil, settingsSvc, nil, nil)svc := NewDashboardService(db, dockerSvc, containerSvc, nil, nil, settingsSvc, nil, nil, nil, nil)defaultSnapshot, err := svc.GetSnapshot(t.Context(), DashboardActionItemsOptions{}, true)diff --git a/backend/internal/database/database.go b/backend/internal/database/database.goindex 44241b6dd..3564192b8 100644--- a/backend/internal/database/database.go+++ b/backend/internal/database/database.go@@ -96,7 +96,8 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOption}var gormDB *gorm.DB- for attempt := 1; attempt <= 3; attempt++ {+ connectStart := time.Now()+ for attempt := 1; ; attempt++ {if err = ctx.Err(); err != nil {return nil, fmt.Errorf("failed to connect to database: %w", err)}@@ -113,13 +114,16 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOptionif err == nil {break}- slog.InfoContext(ctx, "Failed to initialize database", "attempt", attempt)- if attempt < 3 {- select {- case <-time.After(3 * time.Second):- case <-ctx.Done():- return nil, fmt.Errorf("failed to connect to database: %w", ctx.Err())- }+ wait := min(time.Second<<min(attempt-1, 4), 15*time.Second)+ if time.Since(connectStart)+wait > time.Minute {+ err = fmt.Errorf("gave up after %d attempts: %w", attempt, err)+ break+ }+ slog.InfoContext(ctx, "Failed to initialize database", "attempt", attempt, "retryIn", wait, "error", err)+ select {+ case <-time.After(wait):+ case <-ctx.Done():+ return nil, fmt.Errorf("failed to connect to database: %w", ctx.Err())}}if err != nil {@@ -161,16 +165,15 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOptionif err != nil {return nil, fmt.Errorf("failed to register database instrumentation: %w", err)}+ sqlDB.SetMaxIdleConns(kit.Ternary(dbProvider == dbProviderPostgres, 15, 5))+ sqlDB.SetMaxOpenConns(kit.Ternary(dbProvider == dbProviderPostgres, 50, 20))+ sqlDB.SetConnMaxLifetime(5 * time.Minute)+ sqlDB.SetConnMaxIdleTime(3 * time.Minute)metrics.ReportDBStatsMetrics(sqlDB)if migrationErr := migrateDatabase(ctx, sqlDB, dbProvider, options, latestMigrationVersion); migrationErr != nil {slog.ErrorContext(ctx, "Failed to run migrations", "error", migrationErr)return nil, fmt.Errorf("failed to run migrations: %w", migrationErr)}-- sqlDB.SetMaxIdleConns(kit.Ternary(dbProvider == dbProviderPostgres, 15, 5))- sqlDB.SetMaxOpenConns(kit.Ternary(dbProvider == dbProviderPostgres, 50, 20))- sqlDB.SetConnMaxLifetime(5 * time.Minute)- sqlDB.SetConnMaxIdleTime(3 * time.Minute)return db, nil}@@ -452,8 +455,16 @@ func ParseSQLiteConnectionString(connString string) (string, error) {qs := sqliteutil.NormalizeQuery(connStringUrl.Query())// Foreign-key enforcement is required on every pooled connection.qs.Add("_pragma", "foreign_keys(1)")- connStringUrl.RawQuery = qs.Encode()- return connStringUrl.String(), nil+ return (&url.URL{+ Scheme: connStringUrl.Scheme,+ Opaque: connStringUrl.Opaque,+ User: connStringUrl.User,+ Host: connStringUrl.Host,+ Path: connStringUrl.Path,+ RawPath: connStringUrl.RawPath,+ RawQuery: qs.Encode(),+ Fragment: connStringUrl.Fragment,+ }).String(), nil}// FindEnvironmentIDByApiKey finds the environment ID that is associated with the given API key.diff --git a/backend/internal/di/di.go b/backend/internal/di/di.goindex 776c1e6d0..69f4a67a3 100644--- a/backend/internal/di/di.go+++ b/backend/internal/di/di.go@@ -4,6 +4,7 @@ package diimport ("context""log/slog"+ "net/http""go.uber.org/fx"@@ -14,16 +15,20 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/build"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/dashboard""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/federated""github.com/getarcaneapp/arcane/backend/v2/internal/gitops""github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"+ "github.com/getarcaneapp/arcane/backend/v2/internal/health""github.com/getarcaneapp/arcane/backend/v2/internal/image"+ "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate""github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/network"@@ -32,7 +37,9 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/passkey""github.com/getarcaneapp/arcane/backend/v2/internal/port""github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/role"+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3""github.com/getarcaneapp/arcane/backend/v2/internal/search""github.com/getarcaneapp/arcane/backend/v2/internal/session""github.com/getarcaneapp/arcane/backend/v2/internal/settings"@@ -43,6 +50,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/upload""github.com/getarcaneapp/arcane/backend/v2/internal/user""github.com/getarcaneapp/arcane/backend/v2/internal/variable"+ "github.com/getarcaneapp/arcane/backend/v2/internal/version""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability""github.com/getarcaneapp/arcane/backend/v2/internal/webhook"@@ -53,10 +61,10 @@ import (// ActorOptions provides the durable Francis runtime and admission.var ActorOptions = fx.Options(- fx.Provide(provideActorRuntimeInternal, provideRunCoordinatorInternal),- fx.Provide(provideAdmissionGateInternal),- fx.Provide(provideWorkflowEngineInternal),- fx.Provide(provideTunnelRegistryInternal),+ fx.Provide(provideActorRuntime, provideRunCoordinator),+ fx.Provide(provideAdmissionGate),+ fx.Provide(provideWorkflowEngine),+ fx.Provide(provideTunnelRegistry),)// ServiceOptions constructs services once and injects them into route modules.@@ -66,7 +74,7 @@ var ServiceOptions = fx.Options(),fx.Provide(fx.Annotate(activity.NewActivityService, fx.As(fx.Self()), fx.As(new(activitylib.Service))),- event.NewEventService,+ fx.Annotate(event.NewEventService, fx.OnStop(func(ctx context.Context, service *event.EventService) error { return service.Stop(ctx) })),job.NewJobService,role.NewRoleService,apikey.NewApiKeyService,@@ -81,9 +89,9 @@ var ServiceOptions = fx.Options(environment.NewEnvironmentService,apns.NewApnsService,notification.NewNotificationService,- provideVulnerabilityServiceInternal,- provideImageUpdateServiceInternal,- provideImageServiceInternal,+ provideVulnerabilityService,+ provideImageUpdateService,+ provideImageService,build.NewBuildService,project.NewLifecycleService,container.NewContainerService,@@ -93,16 +101,17 @@ var ServiceOptions = fx.Options(swarm.NewSwarmService,template.NewTemplateService,oidc.NewOidcService,- provideSystemServiceInternal,+ provideSystemService,diagnostics.NewDiagnosticsService,- provideGitOpsSyncServiceInternal,+ provideGitOpsSyncService,variable.NewVariableService,backup.NewRecoveryKeyStore,upload.NewUploadService,+ health.NewService,auth.NewAuthService,settings.NewSettingsSearchService,fx.Annotate(settings.NewSettingsService, fx.OnStop(func(ctx context.Context, service *settings.SettingsService) error { return service.Stop(ctx) })),- provideVolumeServiceInternal,+ provideVolumeService,fx.Annotate(webhook.NewWebhookService,fx.OnStart(func(ctx context.Context, service *webhook.WebhookService) error { return service.LoadTokenHashes(ctx) }),fx.OnStop(func(ctx context.Context, service *webhook.WebhookService) {@@ -114,16 +123,45 @@ var ServiceOptions = fx.Options(),// These providers bind config values, callbacks, or managed resource lifetimes.fx.Provide(- provideJWKSetManagerInternal,- provideDockerClientServiceInternal,- provideProjectServiceInternal,- provideBackupEngineInternal,- provideVersionServiceInternal,- provideGitRepositoryServiceInternal,- provideS3ServiceInternal,- provideContainerRegistryServiceInternal,- provideUpdaterServiceInternal,- provideAuthMiddlewareInternal,+ provideJWKSetManager,+ provideDockerClientService,+ func(+ db *database.DB,+ localSettings *settings.SettingsService,+ localEvent *event.EventService,+ localImage *image.ImageService,+ localDocker *docker.DockerClientService,+ localBuild *build.BuildService,+ lifecycleService *project.LifecycleService,+ localKv *kv.KVService,+ localRegistry *registry.ContainerRegistryService,+ localEnvironment *environment.EnvironmentService,+ cfg *config.Config,+ ) *project.ProjectService {+ return project.NewProjectService(db, localSettings, localEvent, localImage, localDocker, localBuild, lifecycleService, localRegistry, cfg, localKv, localEnvironment.GetEnabledRegistryCredentials)+ },+ provideBackupEngine,+ func(+ httpClient *http.Client,+ cfg *config.Config,+ localRegistry *registry.ContainerRegistryService,+ localDocker *docker.DockerClientService,+ imageUpdate *imageupdate.ImageUpdateService,+ settingsService *settings.SettingsService,+ ) *version.VersionService {+ return version.NewVersionService(httpClient, cfg.UpdateCheckDisabled, config.Version, config.Revision, localRegistry, localDocker, imageUpdate, settingsService)+ },+ func(db *database.DB, cfg *config.Config, eventService *event.EventService, settingsService *settings.SettingsService) *gitrepo.GitRepositoryService {+ return gitrepo.NewGitRepositoryService(db, cfg.GitWorkDir, eventService, settingsService)+ },+ func(db *database.DB, localEnvironment *environment.EnvironmentService) *s3domain.S3DestinationService {+ return s3domain.NewS3DestinationService(db, localEnvironment.CheckS3DestinationReferences)+ },+ func(db *database.DB, localDocker *docker.DockerClientService, kvService *kv.KVService, settingsService *settings.SettingsService) *registry.ContainerRegistryService {+ return registry.NewContainerRegistryService(db, func(ctx context.Context) (registry.RegistryDaemonClient, error) { return localDocker.GetClient(ctx) }, kvService, settingsService)+ },+ provideUpdaterService,+ provideAuthMiddleware,),fx.Provide(event.New,@@ -149,13 +187,34 @@ var ServiceOptions = fx.Options(variable.New,upload.New,search.New,- provideActivityModuleInternal,- provideImageUpdateModuleInternal,- provideSettingsModuleInternal,- provideS3ModuleInternal,- provideContainerRegistryModuleInternal,- provideAuthModuleInternal,- provideUserModuleInternal,+ health.New,+ func(service *activity.ActivityService, localEnvironment *environment.EnvironmentService) *activity.Module {+ return activity.New(service, activity.EnvironmentDependencies{+ ProxyJSONRequest: localEnvironment.ProxyJSONRequest,+ ListActiveRemoteEnvironments: localEnvironment.ListActiveRemoteEnvironments,+ GetActiveRemoteEnvironment: localEnvironment.GetActiveRemoteEnvironmentSnapshot,+ ProxyJSONRequestForEnvironment: localEnvironment.ProxyJSONRequestForEnvironment,+ ResolveEnvironmentName: localEnvironment.ResolveEnvironmentName,+ })+ },+ func(service *imageupdate.ImageUpdateService, imageService *image.ImageService) *imageupdate.Module {+ return imageupdate.New(service, imageService.GetUpdateInfoByImageRefs)+ },+ func(service *settings.SettingsService, search *settings.SettingsSearchService, localEnvironment *environment.EnvironmentService, cfg *config.Config) *settings.Module {+ return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)+ },+ func(service *s3domain.S3DestinationService, localEnvironment *environment.EnvironmentService) *s3domain.Module {+ return s3domain.New(service, localEnvironment.SyncS3DestinationsToRemoteEnvironments)+ },+ func(service *registry.ContainerRegistryService, localEnvironment *environment.EnvironmentService) *registry.Module {+ return registry.New(service, localEnvironment.SyncRegistriesToRemoteEnvironments)+ },+ func(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {+ return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)+ },+ func(service *user.UserService, localAuth *auth.AuthService, settingsService *settings.SettingsService) *user.Module {+ return user.New(service, localAuth.InvalidateUserTokenCache, settingsService)+ },),)@@ -171,7 +230,7 @@ var JobOptions = fx.Options(scheduler.NewPruningVolumeHelperJob,scheduler.NewExpiredSessionsCleanupJob,scheduler.NewScheduledPruneJob,- provideFilesystemWatcherJobInternal,+ provideFilesystemWatcherJob,fx.Annotate(scheduler.NewVulnerabilityScanJob, fx.ResultTags(`name:"vulnerability-scan"`)),scheduler.NewVulnerabilityRiskJob,fx.Annotate(scheduler.NewAutoPatchJob, fx.ResultTags(`name:"auto-patch"`)),diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.goindex 0da1a084e..778136061 100644--- a/backend/internal/di/di_test.go+++ b/backend/internal/di/di_test.go@@ -143,7 +143,7 @@ func TestOptionsValidate(t *testing.T) {require.NoError(t, err)}-func TestDockerEventLifecycleInternal(t *testing.T) {+func TestDockerEventLifecycle(t *testing.T) {for _, mode := range []string{"manager", "agent", "edge"} {t.Run(mode, func(t *testing.T) {streamStopped := make(chan struct{})@@ -188,7 +188,7 @@ func TestDockerEventLifecycleInternal(t *testing.T) {cfg := &config.Config{DockerHost: server.URL, AgentMode: mode != "manager", EdgeAgent: mode == "edge"}lifecycle := fxtest.NewLifecycle(t)eventService := event.NewEventService(databaseDB, cfg, nil)- provideDockerClientServiceInternal(t.Context(), lifecycle, databaseDB, cfg, nil, eventService)+ provideDockerClientService(t.Context(), lifecycle, databaseDB, cfg, nil, eventService)t.Cleanup(func() {ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 3*time.Second)defer cancel()diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex 46e3fe233..7c0e390d6 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -18,7 +18,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/apikey""github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/backup"- "github.com/getarcaneapp/arcane/backend/v2/internal/build""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/container""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -32,7 +31,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/network""github.com/getarcaneapp/arcane/backend/v2/internal/notification"- "github.com/getarcaneapp/arcane/backend/v2/internal/passkey""github.com/getarcaneapp/arcane/backend/v2/internal/project""github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/role"@@ -42,7 +40,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/system""github.com/getarcaneapp/arcane/backend/v2/internal/template""github.com/getarcaneapp/arcane/backend/v2/internal/updater"- "github.com/getarcaneapp/arcane/backend/v2/internal/user""github.com/getarcaneapp/arcane/backend/v2/internal/version""github.com/getarcaneapp/arcane/backend/v2/internal/volume""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"@@ -56,25 +53,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk")-func provideActivityModuleInternal(service *activity.ActivityService, localEnvironment *environment.EnvironmentService) *activity.Module {- return activity.New(service, activity.EnvironmentDependencies{- ProxyJSONRequest: localEnvironment.ProxyJSONRequest,- ListActiveRemoteEnvironments: localEnvironment.ListActiveRemoteEnvironments,- GetActiveRemoteEnvironment: localEnvironment.GetActiveRemoteEnvironmentSnapshot,- ProxyJSONRequestForEnvironment: localEnvironment.ProxyJSONRequestForEnvironment,- ResolveEnvironmentName: localEnvironment.ResolveEnvironmentName,- })-}--func provideImageUpdateModuleInternal(service *imageupdate.ImageUpdateService, imageService *image.ImageService) *imageupdate.Module {- return imageupdate.New(service, imageService.GetUpdateInfoByImageRefs)-}--func provideSettingsModuleInternal(service *settings.SettingsService, search *settings.SettingsSearchService, localEnvironment *environment.EnvironmentService, cfg *config.Config) *settings.Module {- return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)-}--func provideBackupEngineInternal(lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, roles *role.RoleService, cfg *config.Config) *backup.Engine {+func provideBackupEngine(lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, roles *role.RoleService, cfg *config.Config) *backup.Engine {engine := backup.NewEngine(admission, imageService)engine.SetAuthorize(func(ctx context.Context, requester backuptypes.Requester) error {if cfg.AgentMode && requester.UserID == "agent" && requester.APIKeyID == "" {@@ -99,7 +78,7 @@ func provideBackupEngineInternal(lc fx.Lifecycle, admission *runs.Admission, imareturn engine}-func provideActorRuntimeInternal(+func provideActorRuntime(appCtx context.Context,cfg *config.Config,db *database.DB,@@ -130,7 +109,7 @@ func provideActorRuntimeInternal(return runtime, nil}-func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVService, cfg *config.Config) (*runs.Coordinator, error) {+func provideRunCoordinator(runtime *francis.Runtime, store *kv.KVService, cfg *config.Config) (*runs.Coordinator, error) {coordinator := runs.New(store, runtime.Service(), cfg.GetLocation())if err := coordinator.Register(runtime); err != nil {return nil, err@@ -138,9 +117,9 @@ func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVServicereturn coordinator, nil}-// provideWorkflowEngineInternal starts monitoring after the host is ready; its+// provideWorkflowEngine starts monitoring after the host is ready; its// hook follows the runtime's because the engine depends on it.-func provideWorkflowEngineInternal(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities *activity.ActivityService, lc fx.Lifecycle) (*flow.Engine, error) {+func provideWorkflowEngine(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities *activity.ActivityService, lc fx.Lifecycle) (*flow.Engine, error) {engine, err := flow.New(appCtx, runtime, coordinator, activities)if err != nil {return nil, err@@ -149,7 +128,7 @@ func provideWorkflowEngineInternal(appCtx context.Context, runtime *francis.Runtreturn engine, nil}-func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, error) {+func provideAdmissionGate(runtime *francis.Runtime) (*runs.Admission, error) {admission := runs.NewAdmission(runtime.Service(), uuid.New().String())if err := admission.Register(runtime); err != nil {return nil, err@@ -157,7 +136,7 @@ func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, erreturn admission, nil}-func provideTunnelRegistryInternal(lc fx.Lifecycle) *edge.TunnelRegistry {+func provideTunnelRegistry(lc fx.Lifecycle) *edge.TunnelRegistry {localRegistry := edge.NewTunnelRegistry()edge.SetDefaultRegistry(localRegistry)lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {@@ -167,7 +146,7 @@ func provideTunnelRegistryInternal(lc fx.Lifecycle) *edge.TunnelRegistry {return localRegistry}-func provideDockerClientServiceInternal(+func provideDockerClientService(ctx context.Context,lc fx.Lifecycle,db *database.DB,@@ -219,69 +198,12 @@ func provideDockerClientServiceInternal(return service}-func provideVersionServiceInternal(- httpClient *http.Client,- cfg *config.Config,- localRegistry *registry.ContainerRegistryService,- localDocker *docker.DockerClientService,- imageUpdate *imageupdate.ImageUpdateService,- settingsService *settings.SettingsService,-) *version.VersionService {- return version.NewVersionService(httpClient, cfg.UpdateCheckDisabled, config.Version, config.Revision, localRegistry, localDocker, imageUpdate, settingsService)-}--func provideGitRepositoryServiceInternal(db *database.DB, cfg *config.Config, eventService *event.EventService, settingsService *settings.SettingsService) *gitrepo.GitRepositoryService {- return gitrepo.NewGitRepositoryService(db, cfg.GitWorkDir, eventService, settingsService)-}--func provideS3ModuleInternal(service *s3domain.S3DestinationService, localEnvironment *environment.EnvironmentService) *s3domain.Module {- return s3domain.New(service, localEnvironment.SyncS3DestinationsToRemoteEnvironments)-}--func provideS3ServiceInternal(db *database.DB, localEnvironment *environment.EnvironmentService) *s3domain.S3DestinationService {- return s3domain.NewS3DestinationService(db, localEnvironment.CheckS3DestinationReferences)-}--func provideAuthModuleInternal(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {- return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)-}--func provideContainerRegistryServiceInternal(- db *database.DB,- localDocker *docker.DockerClientService,- kvService *kv.KVService,- settingsService *settings.SettingsService,-) *registry.ContainerRegistryService {- return registry.NewContainerRegistryService(db, func(ctx context.Context) (registry.RegistryDaemonClient, error) { return localDocker.GetClient(ctx) }, kvService, settingsService)-}--func provideContainerRegistryModuleInternal(service *registry.ContainerRegistryService, localEnvironment *environment.EnvironmentService) *registry.Module {- return registry.New(service, localEnvironment.SyncRegistriesToRemoteEnvironments)-}--func provideProjectServiceInternal(- db *database.DB,- localSettings *settings.SettingsService,- localEvent *event.EventService,- localImage *image.ImageService,- localDocker *docker.DockerClientService,- localBuild *build.BuildService,- lifecycleService *project.LifecycleService,- localKv *kv.KVService,- localRegistry *registry.ContainerRegistryService,- localEnvironment *environment.EnvironmentService,- cfg *config.Config,-) *project.ProjectService {- return project.NewProjectService(db, localSettings, localEvent, localImage, localDocker, localBuild, lifecycleService, localRegistry, cfg, localKv, localEnvironment.GetEnabledRegistryCredentials)-}-// updaterServiceParams includes actor registration and worker lifecycle dependencies.type updaterServiceParams struct {fx.InConfig *config.ConfigWorkflows *flow.Engine- Coordinator *runs.CoordinatorAdmission *runs.AdmissionRoles *role.RoleServiceDB *database.DB@@ -297,8 +219,8 @@ type updaterServiceParams struct {Activity *activity.ActivityService}-// provideImageUpdateServiceInternal registers the image-check workflow while the host is still unstarted.-func provideImageUpdateServiceInternal(+// provideImageUpdateService registers the image-check workflow while the host is still unstarted.+func provideImageUpdateService(db *database.DB,settingsService *settings.SettingsService,registryService *registry.ContainerRegistryService,@@ -315,8 +237,8 @@ func provideImageUpdateServiceInternal(return service, nil}-// provideImageServiceInternal registers the auto-patch workflow while the host is still unstarted.-func provideImageServiceInternal(+// provideImageService registers the auto-patch workflow while the host is still unstarted.+func provideImageService(db *database.DB,dockerService *docker.DockerClientService,registryService *registry.ContainerRegistryService,@@ -334,8 +256,8 @@ func provideImageServiceInternal(return service, nil}-// provideVulnerabilityServiceInternal registers the scheduled scan workflow while the host is still unstarted.-func provideVulnerabilityServiceInternal(+// provideVulnerabilityService registers the scheduled scan workflow while the host is still unstarted.+func provideVulnerabilityService(db *database.DB,cfg *config.Config,dockerService *docker.DockerClientService,@@ -355,7 +277,7 @@ func provideVulnerabilityServiceInternal(return service, nil}-func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterService, error) {+func provideUpdaterService(p updaterServiceParams) (*updater.UpdaterService, error) {service, err := updater.NewUpdaterService(p.DB,p.Settings,@@ -369,7 +291,6 @@ func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterServp.System,p.Activity,p.Config,- p.Coordinator,p.Admission,p.Roles,)@@ -382,17 +303,13 @@ func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterServreturn service, nil}-func provideUserModuleInternal(service *user.UserService, localAuth *auth.AuthService, settingsService *settings.SettingsService) *user.Module {- return user.New(service, localAuth.InvalidateUserTokenCache, settingsService)-}--func provideJWKSetManagerInternal(ctx context.Context, lc fx.Lifecycle) *oidcjwk.KeySetManager {+func provideJWKSetManager(ctx context.Context, lc fx.Lifecycle) *oidcjwk.KeySetManager {manager := oidcjwk.NewKeySetManager(ctx)lc.Append(fx.Hook{OnStop: manager.Shutdown})return manager}-func provideAuthMiddlewareInternal(+func provideAuthMiddleware(authService *auth.AuthService,apiKey *apikey.ApiKeyService,env *environment.EnvironmentService,@@ -405,7 +322,7 @@ func provideAuthMiddlewareInternal(WithPermissionResolver(localRole)}-func provideFilesystemWatcherJobInternal(+func provideFilesystemWatcherJob(ctx context.Context,lc fx.Lifecycle,localProject *project.ProjectService,@@ -461,8 +378,8 @@ func provideFilesystemWatcherJobInternal(return job, nil}-// provideGitOpsSyncServiceInternal registers the gitops-sync workflow while the host is still unstarted.-func provideGitOpsSyncServiceInternal(+// provideGitOpsSyncService registers the gitops-sync workflow while the host is still unstarted.+func provideGitOpsSyncService(db *database.DB,repoService *gitrepo.GitRepositoryService,projectService *project.ProjectService,@@ -478,8 +395,8 @@ func provideGitOpsSyncServiceInternal(return service, nil}-// provideVolumeServiceInternal registers the volume backup workflows while the host is still unstarted.-func provideVolumeServiceInternal(+// provideVolumeService registers the volume backup workflows while the host is still unstarted.+func provideVolumeService(lc fx.Lifecycle,db *database.DB,dockerService *docker.DockerClientService,@@ -501,8 +418,8 @@ func provideVolumeServiceInternal(return service, nil}-// provideSystemServiceInternal registers the system backup workflows while the host is still unstarted.-func provideSystemServiceInternal(+// provideSystemService registers the system backup workflows while the host is still unstarted.+func provideSystemService(db *database.DB,dockerService *docker.DockerClientService,containerService *container.ContainerService,diff --git a/backend/internal/docker/service.go b/backend/internal/docker/service.goindex c3cecacfc..e15ccb2d1 100644--- a/backend/internal/docker/service.go+++ b/backend/internal/docker/service.go@@ -77,30 +77,6 @@ func (s *DockerClientService) WithClient(dockerClient *client.Client) *DockerCli}}-func detectDockerAPIVersion(ctx context.Context, host string) (string, error) {- probeClient, err := client.New(client.WithHost(host))- if err != nil {- return "", fmt.Errorf("failed to create Docker probe client: %w", err)- }- defer closeDockerClient(ctx, probeClient, "probe")-- ctx, cancel := context.WithTimeout(ctx, dockerClientNegotiationTimeout)- defer cancel()-- pingResult, err := probeClient.Ping(ctx, client.PingOptions{})- if err != nil {- return "", fmt.Errorf("failed to negotiate Docker API version: %w", err)- }-- apiVersion := strings.TrimSpace(pingResult.APIVersion)- if apiVersion == "" {- slog.WarnContext(ctx, "Docker ping did not report an API version, using minimum supported client API version", "apiVersion", client.MinAPIVersion)- return client.MinAPIVersion, nil- }-- return apiVersion, nil-}-func closeDockerClient(ctx context.Context, cli *client.Client, purpose string) {if cli == nil {return@@ -115,44 +91,40 @@ func closeDockerClient(ctx context.Context, cli *client.Client, purpose string)// It initializes the client on the first call.func (s *DockerClientService) GetClient(ctx context.Context) (*client.Client, error) {s.mu.Lock()- if s.Client != nil {- cli := s.Client- s.mu.Unlock()+ cli := s.Client+ s.mu.Unlock()+ if cli != nil {return cli, nil}- s.mu.Unlock()- apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)- if err != nil {- return nil, fmt.Errorf("failed to create Docker client: %w", err)- }- cli, err := client.New(client.WithHost(s.config.DockerHost), client.WithAPIVersion(apiVersion))- if err != nil {- return nil, fmt.Errorf("failed to create Docker client: failed to configure Docker client API version %s: %w", apiVersion, err)+ if err := s.RefreshClient(ctx); err != nil {+ return nil, err}s.mu.Lock()- if s.Client != nil {- existingClient := s.Client- s.mu.Unlock()- closeDockerClient(ctx, cli, "unused after concurrent initialization")- return existingClient, nil- }-- s.Client = cli- s.clientVersion = cli.ClientVersion()- s.clientLastProbe = time.Now()- s.mu.Unlock()-- return cli, nil+ defer s.mu.Unlock()+ return s.Client, nil}// RefreshClient probes the Docker daemon and recreates the cached client when// the daemon's effective API version changed.func (s *DockerClientService) RefreshClient(ctx context.Context) error {- apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)+ probeClient, err := client.New(client.WithHost(s.config.DockerHost))if err != nil {- return fmt.Errorf("failed to refresh Docker client: %w", err)+ return fmt.Errorf("failed to refresh Docker client: failed to create Docker probe client: %w", err)+ }+ pingCtx, cancel := context.WithTimeout(ctx, dockerClientNegotiationTimeout)+ pingResult, err := probeClient.Ping(pingCtx, client.PingOptions{})+ cancel()+ closeDockerClient(ctx, probeClient, "probe")+ if err != nil {+ return fmt.Errorf("failed to refresh Docker client: failed to negotiate Docker API version: %w", err)+ }++ apiVersion := strings.TrimSpace(pingResult.APIVersion)+ if apiVersion == "" {+ slog.WarnContext(ctx, "Docker ping did not report an API version, using minimum supported client API version", "apiVersion", client.MinAPIVersion)+ apiVersion = client.MinAPIVersion}s.mu.Lock()@@ -219,11 +191,15 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {eventBackoff.InitialInterval = 500 * time.MillisecondeventBackoff.MaxInterval = 30 * time.Second+ // Resume from the last received event so a reconnect replays events missed during the gap.+ var lastEventNano int64for ctx.Err() == nil {if dockerClient, err := s.GetClient(ctx); err != nil {slog.WarnContext(ctx, "failed to connect to Docker event stream", "error", err)} else {- result := dockerClient.Events(ctx, client.EventsListOptions{})+ result := dockerClient.Events(ctx, client.EventsListOptions{+ Since: kit.Ternary(lastEventNano == 0, "", fmt.Sprintf("%d.%09d", lastEventNano/int64(time.Second), lastEventNano%int64(time.Second))),+ })streamStart := time.Now()var streamErr errorstream:@@ -239,6 +215,7 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {if msg.TimeNano == 0 {msg.TimeNano = time.Now().UnixNano()}+ lastEventNano = msg.TimeNanos.EventBus().Publish(msg)case recvErr, ok := <-result.Err:streamErr = kit.Ternary(ok, recvErr, nil)@@ -261,16 +238,21 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {returncase <-timer.C:}+ // A restarted daemon may report a different API version, so re-probe before reconnecting.+ if err := s.RefreshClient(ctx); err != nil {+ slog.DebugContext(ctx, "failed to refresh Docker client before event stream reconnect", "error", err)+ }}}-// listCoalesced shares one detached op among concurrent callers; each waiter stops on its own+// ListCoalesced shares one detached Call among concurrent callers; each waiter stops on its own// context, nothing is cached, and the shared result must be treated as immutable.-func listCoalesced[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {+// Streaming calls (logs, events, attach, pull, exec, stats, wait) must not use it because Call applies a deadline.+func (s *DockerClientService) ListCoalesced[T any](ctx context.Context, group *singleflight.Group, op func(context.Context, *client.Client) (T, error)) (T, error) {var zero Tch := group.DoChan("list", func() (any, error) {- return op(context.WithoutCancel(ctx))+ return s.Call(context.WithoutCancel(ctx), op)})select {@@ -288,8 +270,9 @@ func listCoalesced[T any](ctx context.Context, group *singleflight.Group, op fun}}-// callAPI runs op against the cached client under the configured Docker API timeout.-func callAPI[T any](ctx context.Context, s *DockerClientService, op func(context.Context, *client.Client) (T, error)) (T, error) {+// Call runs op against the cached client under the configured Docker API timeout.+// Streaming calls (logs, events, attach, pull, exec, stats, wait) must not use it because it applies a deadline.+func (s *DockerClientService) Call[T any](ctx context.Context, op func(context.Context, *client.Client) (T, error)) (T, error) {dockerClient, err := s.GetClient(ctx)if err != nil {var zero T@@ -305,36 +288,22 @@ func callAPI[T any](ctx context.Context, s *DockerClientService, op func(context}func (s *DockerClientService) ListContainers(ctx context.Context) ([]container.Summary, error) {- return listCoalesced(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {- return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {- containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})- if err != nil {- return nil, fmt.Errorf("failed to list Docker containers: %w", err)- }- return containerList.Items, nil- })+ return s.ListCoalesced(ctx, &s.containerListFlight, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {+ containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ if err != nil {+ return nil, fmt.Errorf("failed to list Docker containers: %w", err)+ }+ return containerList.Items, nil})}func (s *DockerClientService) ListImages(ctx context.Context) ([]image.Summary, error) {- return listCoalesced(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {- return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]image.Summary, error) {- imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{All: true})- if err != nil {- return nil, fmt.Errorf("failed to list Docker images: %w", err)- }- return imageList.Items, nil- })- })-}--func (s *DockerClientService) listNetworks(ctx context.Context) ([]network.Summary, error) {- return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {- networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ return s.ListCoalesced(ctx, &s.imageListFlight, func(ctx context.Context, dockerClient *client.Client) ([]image.Summary, error) {+ imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{All: true})if err != nil {- return nil, fmt.Errorf("failed to list Docker networks: %w", err)+ return nil, fmt.Errorf("failed to list Docker images: %w", err)}- return networkList.Items, nil+ return imageList.Items, nil})}@@ -364,7 +333,13 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*ddefer utils.RecoverToError(&workerErr, "docker snapshot worker")var err error- networks, err = s.listNetworks(groupCtx)+ networks, err = s.Call(groupCtx, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {+ networkList, listErr := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ if listErr != nil {+ return nil, fmt.Errorf("failed to list Docker networks: %w", listErr)+ }+ return networkList.Items, nil+ })if err != nil {slog.WarnContext(groupCtx, "failed to list Docker networks for snapshot", "error", err)}@@ -374,7 +349,7 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*ddefer utils.RecoverToError(&workerErr, "docker snapshot worker")var err error- volumes, err = callAPI(groupCtx, s, func(ctx context.Context, dockerClient *client.Client) (*client.VolumeListResult, error) {+ volumes, err = s.Call(groupCtx, func(ctx context.Context, dockerClient *client.Client) (*client.VolumeListResult, error) {volResp, listErr := dockerClient.VolumeList(ctx, client.VolumeListOptions{})if listErr != nil {return nil, fmt.Errorf("failed to list Docker volumes: %w", listErr)@@ -476,7 +451,13 @@ func (s *DockerClientService) GetAllNetworks(ctx context.Context) ([]network.Sumif err != nil {return nil, 0, 0, 0, err}- networks, err := s.listNetworks(ctx)+ networks, err := s.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {+ networkList, listErr := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ if listErr != nil {+ return nil, fmt.Errorf("failed to list Docker networks: %w", listErr)+ }+ return networkList.Items, nil+ })if err != nil {return nil, 0, 0, 0, err}diff --git a/backend/internal/docker/service_test.go b/backend/internal/docker/service_test.goindex e1e537ceb..fd803a156 100644--- a/backend/internal/docker/service_test.go+++ b/backend/internal/docker/service_test.go@@ -274,6 +274,62 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStream(t *testing.T) {}, time.Second, time.Millisecond)}+func TestDockerClientService_WatchEventsResumesFromLastEventAfterRefresh(t *testing.T) {+ first := events.Message{+ Type: events.ImageEventType,+ Action: events.ActionPull,+ TimeNano: 1_700_000_000_123_456_789,+ }+ payload, err := json.Marshal(first)+ require.NoError(t, err)+ var pings, eventCalls atomic.Int32+ type reconnect struct {+ since string+ pings int32+ }+ reconnected := make(chan reconnect, 1)+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {+ switch path.Base(r.URL.Path) {+ case "_ping":+ pings.Add(1)+ w.Header().Set("Api-Version", "1.41")+ w.WriteHeader(http.StatusOK)+ case "events":+ w.Header().Set("Content-Type", "application/json")+ if eventCalls.Add(1) == 1 {+ assert.Empty(t, r.URL.Query().Get("since"))+ _, _ = w.Write(payload)+ return+ }+ select {+ case reconnected <- reconnect{since: r.URL.Query().Get("since"), pings: pings.Load()}:+ default:+ }+ w.WriteHeader(http.StatusOK)+ <-r.Context().Done()+ default:+ http.NotFound(w, r)+ }+ })+ svc := newDockerClientServiceForTest(t, server.URL)+ t.Cleanup(svc.Close)+ stop, err := concurrency.StartSupervised(t.Context(), "Docker event watcher", func(ctx context.Context) error {+ svc.WatchEvents(ctx)+ return nil+ })+ require.NoError(t, err)+ t.Cleanup(func() { _ = stop(context.WithoutCancel(t.Context())) })++ select {+ case got := <-reconnected:+ assert.Equal(t, "1700000000.123456789", got.since)+ // One probe from the initial GetClient, one from the RefreshClient before reconnecting.+ assert.GreaterOrEqual(t, got.pings, int32(2))+ case <-time.After(5 * time.Second):+ t.Fatal("event stream did not reconnect")+ }+}+func TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {images := []image.Summary{{ID: "sha256:image-a", Containers: -1, Size: 100},diff --git a/backend/internal/environment/children/health/service.go b/backend/internal/environment/children/health/service.goindex 59e4914b8..931c108e2 100644--- a/backend/internal/environment/children/health/service.go+++ b/backend/internal/environment/children/health/service.go@@ -34,11 +34,9 @@ func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customURreqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)defer cancel()- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return "offline", fmt.Errorf("failed to connect to Docker: %w", err)- }- if _, pingErr := dockerClient.Ping(reqCtx, client.PingOptions{}); pingErr != nil {+ if _, pingErr := s.dockerService.Call(reqCtx, func(ctx context.Context, cli *client.Client) (client.PingResult, error) {+ return cli.Ping(ctx, client.PingOptions{})+ }); pingErr != nil {return "offline", fmt.Errorf("docker ping failed: %w", pingErr)}return "online", nildiff --git a/backend/internal/event/service.go b/backend/internal/event/service.goindex bae2265e0..aa4cc725e 100644--- a/backend/internal/event/service.go+++ b/backend/internal/event/service.go@@ -18,8 +18,6 @@ import ("github.com/getarcaneapp/arcane/types/v2/event""github.com/moby/moby/api/types/events"- "github.com/samber/mo"- "github.com/samber/mo/option""go.getarcane.app/kit/pkg""go.getarcane.app/kit/pkg/mapping""go.getarcane.app/streams/agg"@@ -43,6 +41,12 @@ import (// maxConcurrentEdgeSyncs bounds in-flight event forwards to the manager.const maxConcurrentEdgeSyncs = 16+const (+ daemonEventQueueSize = 1024+ daemonEventBatchSize = 64+ daemonEventFlushInterval = 250 * time.Millisecond+)+type EventService struct {changes *concurrency.Signal[struct{}]correlation *correlation.Service@@ -50,6 +54,12 @@ type EventService struct {cfg *config.ConfighttpClient *http.ClientedgeSyncs *semaphore.Weighted+ // Daemon events are persisted in batches by a writer started on first use.+ daemonQueue chan CreateEventRequest+ daemonStart sync.Once+ daemonDone chan struct{}+ daemonMu sync.RWMutex+ daemonStopped bool}func NewEventService(db *database.DB, cfg *config.Config, httpClient *http.Client) *EventService {@@ -65,6 +75,8 @@ func NewEventService(db *database.DB, cfg *config.Config, httpClient *http.Cliencfg: cfg,httpClient: httpClient,edgeSyncs: semaphore.NewWeighted(maxConcurrentEdgeSyncs),+ daemonQueue: make(chan CreateEventRequest, daemonEventQueueSize),+ daemonDone: make(chan struct{}),}}@@ -85,53 +97,62 @@ type CreateEventRequest struct {}func (s *EventService) CreateEvent(ctx context.Context, req CreateEventRequest) (*Event, error) {- severity := cmp.Or(req.Severity, EventSeverityInfo)- userID, username := normalizeEventActor(req.UserID, req.Username)-- eventRecord := &Event{- CreatedAt: time.Now(),- Type: req.Type,- Severity: severity,- Title: req.Title,- Description: req.Description,- ResourceType: req.ResourceType,- ResourceID: req.ResourceID,- ResourceName: req.ResourceName,- UserID: userID,- Username: username,- EnvironmentID: req.EnvironmentID,- Metadata: req.Metadata,- Timestamp: time.Now(),- }-- if req.deduplicationKey != "" {- eventRecord.DeduplicationKey = &req.deduplicationKey+ records, err := s.persistEvents(ctx, []CreateEventRequest{req})+ if err != nil {+ return nil, err}+ return records[0], nil+}- var inserted bool+// persistEvents inserts requests in one transaction, skipping replayed deduplication keys.+func (s *EventService) persistEvents(ctx context.Context, reqs []CreateEventRequest) ([]*Event, error) {+ records := make([]*Event, len(reqs))+ inserted := make([]bool, len(reqs))err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- if req.deduplicationKey != "" {- tx = tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "deduplication_key"}}, DoNothing: true})- }- result := tx.Create(eventRecord)- if result.Error != nil {- return fmt.Errorf("failed to create event: %w", result.Error)+ for index, req := range reqs {+ // A missing actor half mirrors the other; no actor at all is the system.+ userID, username := strings.TrimSpace(kit.FromPtr(req.UserID)), strings.TrimSpace(kit.FromPtr(req.Username))+ records[index] = &Event{+ CreatedAt: time.Now(),+ Type: req.Type,+ Severity: cmp.Or(req.Severity, EventSeverityInfo),+ Title: req.Title,+ Description: req.Description,+ ResourceType: req.ResourceType,+ ResourceID: req.ResourceID,+ ResourceName: req.ResourceName,+ UserID: new(cmp.Or(userID, username, "system")),+ Username: new(cmp.Or(username, userID, "System")),+ EnvironmentID: req.EnvironmentID,+ Metadata: req.Metadata,+ Timestamp: time.Now(),+ DeduplicationKey: kit.Ternary(req.deduplicationKey != "", &req.deduplicationKey, nil),+ }+ create := tx+ if req.deduplicationKey != "" {+ create = tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "deduplication_key"}}, DoNothing: true})+ }+ result := create.Create(records[index])+ if result.Error != nil {+ return fmt.Errorf("failed to create event: %w", result.Error)+ }+ inserted[index] = result.RowsAffected > 0}- inserted = result.RowsAffected > 0return nil})if err != nil {return nil, err}- if !inserted {- return eventRecord, nil+ for index, req := range reqs {+ if !inserted[index] {+ continue+ }+ s.correlateCreatedEvent(req)+ s.changes.Publish(struct{}{})+ s.forwardEventToManager(ctx, records[index])}- s.correlateCreatedEvent(req)- s.changes.Publish(struct{}{})- s.forwardEventToManager(ctx, eventRecord)-- return eventRecord, nil+ return records, nil}// IngestAgentEvent assigns the authenticated environment instead of trusting the agent's local ID.@@ -145,38 +166,30 @@ func (s *EventService) IngestAgentEvent(ctx context.Context, environmentID strin}func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Event) {- if eventModel == nil || s.cfg == nil || !s.cfg.AgentMode {+ if s.cfg == nil || !s.cfg.AgentMode {return}evt := &edge.TunnelEvent{- Type: string(eventModel.Type),- Severity: string(eventModel.Severity),- Title: eventModel.Title,- Description: eventModel.Description,- }- if eventModel.ResourceType != nil {- evt.ResourceType = *eventModel.ResourceType- }- if eventModel.ResourceID != nil {- evt.ResourceID = *eventModel.ResourceID- }- if eventModel.ResourceName != nil {- evt.ResourceName = *eventModel.ResourceName- }- if eventModel.UserID != nil {- evt.UserID = *eventModel.UserID- }- if eventModel.Username != nil {- evt.Username = *eventModel.Username- }- if eventModel.Metadata != nil {- metadataBytes, err := json.Marshal(map[string]any(eventModel.Metadata))- if err != nil {- slog.WarnContext(ctx, "Failed to marshal event metadata for edge sync", "type", eventModel.Type, "error", err)- } else {- evt.MetadataJSON = metadataBytes- }+ Type: string(eventModel.Type),+ Severity: string(eventModel.Severity),+ Title: eventModel.Title,+ Description: eventModel.Description,+ ResourceType: kit.FromPtr(eventModel.ResourceType),+ ResourceID: kit.FromPtr(eventModel.ResourceID),+ ResourceName: kit.FromPtr(eventModel.ResourceName),+ UserID: kit.FromPtr(eventModel.UserID),+ Username: kit.FromPtr(eventModel.Username),+ MetadataJSON: func() []byte {+ if eventModel.Metadata == nil {+ return nil+ }+ metadataBytes, err := json.Marshal(map[string]any(eventModel.Metadata))+ if err != nil {+ slog.WarnContext(ctx, "Failed to marshal event metadata for edge sync", "type", eventModel.Type, "error", err)+ }+ return metadataBytes+ }(),}go func(parentCtx context.Context, outgoing *edge.TunnelEvent) {@@ -194,7 +207,7 @@ func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Evslog.WarnContext(syncCtx, "Failed to sync event to manager over edge tunnel", "type", outgoing.Type, "error", err)return}- if !s.canForwardEventToManagerHTTP() {+ if strings.TrimSpace(s.cfg.AgentToken) == "" || strings.TrimSpace(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl)) == "" {return}if httpErr := s.forwardEventToManagerHTTP(syncCtx, eventModel); httpErr != nil {@@ -205,30 +218,19 @@ func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Ev}(ctx, evt)}-func (s *EventService) canForwardEventToManagerHTTP() bool {- if s.cfg == nil {- return false- }- if strings.TrimSpace(s.cfg.AgentToken) == "" {- return false- }- return strings.TrimSpace(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl)) != ""-}-func (s *EventService) forwardEventToManagerHTTP(ctx context.Context, eventModel *Event) error {- if eventModel == nil {- return errors.New("event is required")+ baseURL, err := url.Parse(strings.TrimSpace(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl)))+ if err != nil {+ return fmt.Errorf("manager API URL is invalid for manager event sync: failed to parse manager API URL: %w", err)}- if s.cfg == nil || strings.TrimSpace(s.cfg.AgentToken) == "" {- return errors.New("agent token is required for manager event sync")+ if baseURL.Scheme != "http" && baseURL.Scheme != "https" {+ return fmt.Errorf("manager API URL is invalid for manager event sync: unsupported scheme %q", baseURL.Scheme)}-- managerEventsURL, err := managerEventEndpointURL(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl))- if err != nil {- return fmt.Errorf("manager API URL is invalid for manager event sync: %w", err)+ if baseURL.Host == "" {+ return errors.New("manager API URL is invalid for manager event sync: manager API URL host is required")}- payload := CreateEventRequest{+ body, err := json.Marshal(CreateEventRequest{Type: eventModel.Type,Severity: eventModel.Severity,Title: eventModel.Title,@@ -239,18 +241,18 @@ func (s *EventService) forwardEventToManagerHTTP(ctx context.Context, eventModelUserID: eventModel.UserID,Username: eventModel.Username,EnvironmentID: eventModel.EnvironmentID,- }-- if len(eventModel.Metadata) > 0 {- payload.Metadata = eventModel.Metadata- }-- body, err := json.Marshal(payload)+ Metadata: eventModel.Metadata,+ })if err != nil {return fmt.Errorf("failed to marshal event payload: %w", err)}- req, err := http.NewRequestWithContext(ctx, http.MethodPost, managerEventsURL, bytes.NewReader(body))+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, (&url.URL{+ Scheme: baseURL.Scheme,+ User: baseURL.User,+ Host: baseURL.Host,+ Path: strings.TrimRight(baseURL.Path, "/") + "/api/events",+ }).String(), bytes.NewReader(body))if err != nil {return fmt.Errorf("failed to create manager event request: %w", err)}@@ -274,62 +276,16 @@ func (s *EventService) forwardEventToManagerHTTP(ctx context.Context, eventModelreturn fmt.Errorf("manager event sync failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))}-func managerEventEndpointURL(rawBaseURL string) (string, error) {- trimmed := strings.TrimSpace(rawBaseURL)- if trimmed == "" {- return "", errors.New("manager API URL is required")- }-- baseURL, err := url.Parse(trimmed)- if err != nil {- return "", fmt.Errorf("failed to parse manager API URL: %w", err)- }- if baseURL.Scheme != "http" && baseURL.Scheme != "https" {- return "", fmt.Errorf("unsupported scheme %q", baseURL.Scheme)- }- if baseURL.Host == "" {- return "", errors.New("manager API URL host is required")- }-- baseURL.RawQuery = ""- baseURL.Fragment = ""- baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/api/events"- return baseURL.String(), nil-}--func normalizeEventActor(userID, username *string) (*string, *string) {- normalizedUserID := normalizeOptionalStringPtr(userID)- normalizedUsername := normalizeOptionalStringPtr(username)-- if normalizedUsername == nil && normalizedUserID != nil {- normalizedUsername = new(*normalizedUserID)- }- if normalizedUserID == nil && normalizedUsername != nil {- normalizedUserID = new(*normalizedUsername)- }- if normalizedUserID == nil && normalizedUsername == nil {- normalizedUserID = new("system")- normalizedUsername = new("System")- }-- return normalizedUserID, normalizedUsername+func (s *EventService) ListEventsPaginated(ctx context.Context, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {+ return paginateEvents(pagination.ApplyFilter(s.db.WithContext(ctx).Model(&Event{}), "environment_id", params.Filters["environmentId"]), params)}-func normalizeOptionalStringPtr(value *string) *string {- if value == nil {- return nil- }- trimmed := strings.TrimSpace(*value)- if trimmed == "" {- return nil- }- return &trimmed+func (s *EventService) GetEventsByEnvironmentPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {+ return paginateEvents(s.db.WithContext(ctx).Model(&Event{}).Where("environment_id = ?", environmentID), params)}-func (s *EventService) ListEventsPaginated(ctx context.Context, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {- var eventRecords []Event- q := s.db.WithContext(ctx).Model(&Event{})-+// paginateEvents applies the shared search and filters to q, then pages and maps the rows.+func paginateEvents(q *gorm.DB, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {if term := strings.TrimSpace(params.Search); term != "" {searchPattern := "%" + term + "%"q = q.Where(@@ -338,42 +294,34 @@ func (s *EventService) ListEventsPaginated(ctx context.Context, params paginatio)}- q = pagination.ApplyFilter(q, "severity", params.Filters["severity"])- q = applyEventTypeFilter(q, params.Filters["type"])- q = pagination.ApplyFilter(q, "resource_type", params.Filters["resourceType"])- q = pagination.ApplyFilter(q, "username", params.Filters["username"])- q = pagination.ApplyFilter(q, "environment_id", params.Filters["environmentId"])-- paginationResp, err := pagination.PaginateAndSortDB(params, q, &eventRecords)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to paginate events: %w", err)+ // Type values with a '.' are exact types; others are category prefixes. Comma-separated values are OR-ed.+ var typeConds []string+ var typeArgs []any+ var exactTypes []string+ for part := range strings.SplitSeq(params.Filters["type"], ",") {+ part = strings.TrimSpace(part)+ switch {+ case part == "":+ case strings.Contains(part, "."):+ exactTypes = append(exactTypes, part)+ default:+ typeConds = append(typeConds, "type LIKE ?")+ typeArgs = append(typeArgs, part+".%")+ }}-- eventDtos, mapErr := mapping.MapSlice[Event, event.Event](eventRecords)- if mapErr != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to map events: %w", mapErr)+ if len(exactTypes) > 0 {+ typeConds = append(typeConds, "type IN ?")+ typeArgs = append(typeArgs, exactTypes)}-- return eventDtos, paginationResp, nil-}--func (s *EventService) GetEventsByEnvironmentPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {- var eventRecords []Event- q := s.db.WithContext(ctx).Model(&Event{}).Where("environment_id = ?", environmentID)-- if term := strings.TrimSpace(params.Search); term != "" {- searchPattern := "%" + term + "%"- q = q.Where(- "title LIKE ? OR description LIKE ? OR COALESCE(resource_name, '') LIKE ? OR COALESCE(username, '') LIKE ?",- searchPattern, searchPattern, searchPattern, searchPattern,- )+ if len(typeConds) > 0 {+ q = q.Where(strings.Join(typeConds, " OR "), typeArgs...)}q = pagination.ApplyFilter(q, "severity", params.Filters["severity"])- q = applyEventTypeFilter(q, params.Filters["type"])q = pagination.ApplyFilter(q, "resource_type", params.Filters["resourceType"])q = pagination.ApplyFilter(q, "username", params.Filters["username"])+ var eventRecords []EventpaginationResp, err := pagination.PaginateAndSortDB(params, q, &eventRecords)if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to paginate events: %w", err)@@ -387,41 +335,6 @@ func (s *EventService) GetEventsByEnvironmentPaginated(ctx context.Context, envireturn eventDtos, paginationResp, nil}-// applyEventTypeFilter filters by event type. Values containing a '.' are-// exact types (e.g. "container.start"); values without one are category-// prefixes (e.g. "container" matches "container.%"). Comma-separated values-// are OR-ed together, mirroring pagination.ApplyFilter's multi-value handling.-func applyEventTypeFilter(q *gorm.DB, value string) *gorm.DB {- if value == "" {- return q- }- var (- exact []string- conds []string- args []any- )- for part := range strings.SplitSeq(value, ",") {- part = strings.TrimSpace(part)- if part == "" {- continue- }- if strings.Contains(part, ".") {- exact = append(exact, part)- } else {- conds = append(conds, "type LIKE ?")- args = append(args, part+".%")- }- }- if len(exact) > 0 {- conds = append(conds, "type IN ?")- args = append(args, exact)- }- if len(conds) == 0 {- return q- }- return q.Where(strings.Join(conds, " OR "), args...)-}-func (s *EventService) GetEventSeverityCounts(ctx context.Context) (event.SeverityCounts, error) {var rows []struct {Severity string@@ -489,60 +402,28 @@ func (s *EventService) DeleteOldEvents(ctx context.Context, olderThan time.Durat}func (s *EventService) LogContainerEvent(ctx context.Context, eventType EventType, containerID, containerName, userID, username, environmentID string, metadata database.JSON) error {- title := s.generateEventTitle(eventType, containerName)- description := s.generateEventDescription(eventType, "container", containerName)- severity := s.getEventSeverity(eventType)-- _, err := s.CreateEvent(ctx, CreateEventRequest{- Type: eventType,- Severity: severity,- Title: title,- Description: description,- ResourceType: new("container"),- ResourceID: new(containerID),- ResourceName: new(containerName),- UserID: new(userID),- Username: new(username),- EnvironmentID: new(environmentID),- Metadata: metadata,- })- return err+ return s.logResourceEvent(ctx, eventType, "container", containerID, containerName, userID, username, environmentID, metadata)}func (s *EventService) LogImageEvent(ctx context.Context, eventType EventType, imageID, imageName, userID, username, environmentID string, metadata database.JSON) error {- title := s.generateEventTitle(eventType, imageName)- description := s.generateEventDescription(eventType, "image", imageName)- severity := s.getEventSeverity(eventType)-- _, err := s.CreateEvent(ctx, CreateEventRequest{- Type: eventType,- Severity: severity,- Title: title,- Description: description,- ResourceType: new("image"),- ResourceID: new(imageID),- ResourceName: new(imageName),- UserID: new(userID),- Username: new(username),- EnvironmentID: new(environmentID),- Metadata: metadata,- })- return err+ return s.logResourceEvent(ctx, eventType, "image", imageID, imageName, userID, username, environmentID, metadata)}func (s *EventService) LogProjectEvent(ctx context.Context, eventType EventType, projectID, projectName, userID, username, environmentID string, metadata database.JSON) error {- title := s.generateEventTitle(eventType, projectName)- description := s.generateEventDescription(eventType, "project", projectName)- severity := s.getEventSeverity(eventType)+ return s.logResourceEvent(ctx, eventType, "project", projectID, projectName, userID, username, environmentID, metadata)+}+// logResourceEvent records a successful operation on a resource using its eventDefinitions entry.+func (s *EventService) logResourceEvent(ctx context.Context, eventType EventType, resourceType, resourceID, resourceName, userID, username, environmentID string, metadata database.JSON) error {+ title, description, severity := describeEvent(eventType, resourceType, resourceName)_, err := s.CreateEvent(ctx, CreateEventRequest{Type: eventType,Severity: severity,Title: title,Description: description,- ResourceType: new("project"),- ResourceID: new(projectID),- ResourceName: new(projectName),+ ResourceType: new(resourceType),+ ResourceID: new(resourceID),+ ResourceName: new(resourceName),UserID: new(userID),Username: new(username),EnvironmentID: new(environmentID),@@ -552,10 +433,7 @@ func (s *EventService) LogProjectEvent(ctx context.Context, eventType EventType,}func (s *EventService) LogUserEvent(ctx context.Context, eventType EventType, userID, username string, metadata database.JSON) error {- title := s.generateEventTitle(eventType, username)- description := s.generateEventDescription(eventType, "user", username)- severity := s.getEventSeverity(eventType)-+ title, description, severity := describeEvent(eventType, "user", username)_, err := s.CreateEvent(ctx, CreateEventRequest{Type: eventType,Severity: severity,@@ -569,45 +447,11 @@ func (s *EventService) LogUserEvent(ctx context.Context, eventType EventType, us}func (s *EventService) LogVolumeEvent(ctx context.Context, eventType EventType, volumeID, volumeName, userID, username, environmentID string, metadata database.JSON) error {- title := s.generateEventTitle(eventType, volumeName)- description := s.generateEventDescription(eventType, "volume", volumeName)- severity := s.getEventSeverity(eventType)-- _, err := s.CreateEvent(ctx, CreateEventRequest{- Type: eventType,- Severity: severity,- Title: title,- Description: description,- ResourceType: new("volume"),- ResourceID: new(volumeID),- ResourceName: new(volumeName),- UserID: new(userID),- Username: new(username),- EnvironmentID: new(environmentID),- Metadata: metadata,- })- return err+ return s.logResourceEvent(ctx, eventType, "volume", volumeID, volumeName, userID, username, environmentID, metadata)}func (s *EventService) LogNetworkEvent(ctx context.Context, eventType EventType, networkID, networkName, userID, username, environmentID string, metadata database.JSON) error {- title := s.generateEventTitle(eventType, networkName)- description := s.generateEventDescription(eventType, "network", networkName)- severity := s.getEventSeverity(eventType)-- _, err := s.CreateEvent(ctx, CreateEventRequest{- Type: eventType,- Severity: severity,- Title: title,- Description: description,- ResourceType: new("network"),- ResourceID: new(networkID),- ResourceName: new(networkName),- UserID: new(userID),- Username: new(username),- EnvironmentID: new(environmentID),- Metadata: metadata,- })- return err+ return s.logResourceEvent(ctx, eventType, "network", networkID, networkName, userID, username, environmentID, metadata)}func (s *EventService) LogErrorEvent(ctx context.Context, eventType EventType, resourceType, resourceID, resourceName, userID, username, environmentID string, err error, metadata database.JSON) {@@ -737,27 +581,13 @@ var eventDefinitions = map[EventType]eventDefinition{EventTypeFederatedExchange: {"Federated credential exchange: %s", "Federated credential exchange for '%s'", EventSeverityInfo},}-func (s *EventService) generateEventTitle(eventType EventType, resourceName string) string {- definition, ok := eventDefinitions[eventType]- return option.Map(func(def eventDefinition) string {- return fmt.Sprintf(def.TitleFormat, resourceName)- })(mo.TupleToOption(definition, ok)).OrElse("Event: " + string(eventType))-}--func (s *EventService) generateEventDescription(eventType EventType, resourceType, resourceName string) string {- definition, ok := eventDefinitions[eventType]- return option.Map(func(def eventDefinition) string {- return fmt.Sprintf(def.DescriptionFormat, resourceName)- })(mo.TupleToOption(definition, ok)).OrElse(- fmt.Sprintf("%s operation performed on %s '%s'", string(eventType), resourceType, resourceName),- )-}--func (s *EventService) getEventSeverity(eventType EventType) EventSeverity {+// describeEvent formats an event's title, description, and severity, with a generic fallback for undefined types.+func describeEvent(eventType EventType, resourceType, resourceName string) (string, string, EventSeverity) {definition, ok := eventDefinitions[eventType]- return option.Map(func(def eventDefinition) EventSeverity {- return def.Severity- })(mo.TupleToOption(definition, ok)).OrElse(EventSeverityInfo)+ if !ok {+ return "Event: " + string(eventType), fmt.Sprintf("%s operation performed on %s '%s'", string(eventType), resourceType, resourceName), EventSeverityInfo+ }+ return fmt.Sprintf(definition.TitleFormat, resourceName), fmt.Sprintf(definition.DescriptionFormat, resourceName), definition.Severity}// RunStreamProducer signals committed event changes so clients can refresh their current query.@@ -866,11 +696,77 @@ func (s *EventService) RecordDockerEvent(ctx context.Context, msg events.Message}req.deduplicationKey = fmt.Sprintf("%x", sha256.Sum256(identity))}+ s.daemonStart.Do(func() { go s.writeDaemonEvents(context.WithoutCancel(ctx)) })+ s.daemonMu.RLock()+ queued := false+ if !s.daemonStopped {+ select {+ case s.daemonQueue <- req:+ queued = true+ default:+ }+ }+ s.daemonMu.RUnlock()+ if queued {+ return+ }+ // A full queue or stopped writer falls back to a direct insert rather than dropping the event.if _, err := s.CreateEvent(ctx, req); err != nil {slog.WarnContext(ctx, "Failed to log Docker daemon event", "type", req.Type, "resourceId", msg.Actor.ID, "error", err)}}+func (s *EventService) writeDaemonEvents(ctx context.Context) {+ defer close(s.daemonDone)+ batch := make([]CreateEventRequest, 0, daemonEventBatchSize)+ var flush <-chan time.Time+ for open := true; open; {+ select {+ case req, ok := <-s.daemonQueue:+ open = ok+ if ok {+ batch = append(batch, req)+ if flush == nil {+ flush = time.After(daemonEventFlushInterval)+ }+ if len(batch) < daemonEventBatchSize {+ continue+ }+ }+ case <-flush:+ }+ if _, err := s.persistEvents(ctx, batch); err != nil {+ slog.WarnContext(ctx, "Failed to persist Docker daemon event batch; retrying individually", "count", len(batch), "error", err)+ for _, req := range batch {+ if _, createErr := s.CreateEvent(ctx, req); createErr != nil {+ slog.WarnContext(ctx, "Failed to log Docker daemon event", "type", req.Type, "resourceId", kit.FromPtr(req.ResourceID), "error", createErr)+ }+ }+ }+ batch = batch[:0]+ flush = nil+ }+}++// Stop flushes queued daemon events; later observations are persisted directly.+func (s *EventService) Stop(ctx context.Context) error {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()+ s.daemonMu.Lock()+ if !s.daemonStopped {+ s.daemonStopped = true+ close(s.daemonQueue)+ }+ s.daemonMu.Unlock()+ s.daemonStart.Do(func() { close(s.daemonDone) })+ select {+ case <-s.daemonDone:+ return nil+ case <-ctx.Done():+ return fmt.Errorf("failed to flush Docker daemon events: %w", ctx.Err())+ }+}+var daemonEventTypes = map[events.Type]map[events.Action]EventType{events.ContainerEventType: {events.ActionCreate: EventTypeContainerCreate,@@ -992,7 +888,7 @@ func (s *EventService) correlateCreatedEvent(req CreateEventRequest) {if req.ResourceType == nil || req.Metadata["source"] == "docker" {return}- if req.EnvironmentID != nil && *req.EnvironmentID != "" && *req.EnvironmentID != "0" {+ if cmp.Or(kit.FromPtr(req.EnvironmentID), "0") != "0" {return}// Failure and observational records do not identify a successful Docker mutation.@@ -1007,12 +903,5 @@ func (s *EventService) correlateCreatedEvent(req CreateEventRequest) {default:return}- var id, name string- if req.ResourceID != nil {- id = *req.ResourceID- }- if req.ResourceName != nil {- name = *req.ResourceName- }- s.MarkDockerExpectation(*req.ResourceType, id, name)+ s.correlation.MarkExpectation(*req.ResourceType, kit.FromPtr(req.ResourceID), kit.FromPtr(req.ResourceName))}diff --git a/backend/internal/event/service_test.go b/backend/internal/event/service_test.goindex 085e2a51a..a3a2e6f26 100644--- a/backend/internal/event/service_test.go+++ b/backend/internal/event/service_test.go@@ -7,6 +7,7 @@ import ("net/http""net/http/httptest""strings"+ "sync""sync/atomic""testing""testing/synctest"@@ -439,6 +440,7 @@ func TestDaemonEventForwardingPreservesMetadata(t *testing.T) {require.True(t, ok)svc.RecordDockerEvent(t.Context(), message)svc.RecordDockerEvent(t.Context(), message)+ require.NoError(t, svc.Stop(t.Context()))select {case payload := <-requests:require.Equal(t, req.Metadata, payload.Metadata)@@ -707,7 +709,7 @@ func TestMapDaemonEvent(t *testing.T) {}}-func TestDaemonEventsPersistEveryOccurrenceImmediately(t *testing.T) {+func TestDaemonEventsPersistEveryOccurrence(t *testing.T) {db := setupEventServiceTestDB(t)svc := NewEventService(db, nil, nil)die := events.Message{Type: events.ContainerEventType, Action: events.ActionDie, Actor: events.Actor{ID: "web", Attributes: map[string]string{"exitCode": "0"}}}@@ -716,8 +718,9 @@ func TestDaemonEventsPersistEveryOccurrenceImmediately(t *testing.T) {for index, msg := range []events.Message{die, stop, die, stop} {msg.TimeNano = int64(index + 1)svc.RecordDockerEvent(t.Context(), msg)- requireDaemonCount(t, db, int64(index+1))}+ require.NoError(t, svc.Stop(t.Context()))+ requireDaemonCount(t, db, 4)var rows []Eventrequire.NoError(t, db.Find(&rows).Error)for _, row := range rows {@@ -746,6 +749,7 @@ func TestDaemonSuppressionAndFailureAllowLaterObservations(t *testing.T) {svc.MarkDockerExpectation("container", "web", "")}svc.RecordDockerEvent(t.Context(), msg)+ require.NoError(t, svc.Stop(t.Context()))requireDaemonCount(t, db, 0)if failure {require.NoError(t, db.Callback().Create().Remove("daemon_failure"))@@ -798,10 +802,14 @@ func TestDockerEventSubscriptionsReadyAndStop(t *testing.T) {done := make(chan error, 1)go func() { done <- run(ctx) }()synctest.Wait()+ time.Sleep(daemonEventFlushInterval)+ synctest.Wait()requireDaemonCount(t, db, 4*occurrences)require.Equal(t, int64(4*occurrences), published.Load())publishBurst(occurrences)synctest.Wait()+ time.Sleep(daemonEventFlushInterval)+ synctest.Wait()requireDaemonCount(t, db, 8*occurrences)require.Equal(t, int64(8*occurrences), published.Load())if closeBus {@@ -811,6 +819,7 @@ func TestDockerEventSubscriptionsReadyAndStop(t *testing.T) {}synctest.Wait()require.NoError(t, <-done)+ require.NoError(t, svc.Stop(t.Context()))cleanup()cleanup()})@@ -818,6 +827,49 @@ func TestDockerEventSubscriptionsReadyAndStop(t *testing.T) {}}+func TestDaemonEventWriterBatchesAndFlushesOnStop(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ db := setupEventServiceTestDB(t)+ sqlDB, err := db.DB.DB()+ require.NoError(t, err)+ defer func() { assert.NoError(t, sqlDB.Close()) }()+ sqlDB.SetMaxOpenConns(1)+ svc := NewEventService(db, nil, nil)+ defer func() { assert.NoError(t, svc.Stop(t.Context())) }()+ var published atomic.Int64+ stopChanges := svc.changes.Subscribe(func(struct{}) { published.Add(1) })+ defer stopChanges()+ record := func(timeNano int64) {+ svc.RecordDockerEvent(t.Context(), events.Message{Type: events.ContainerEventType, Action: events.ActionStart, TimeNano: timeNano, Actor: events.Actor{ID: "web"}})+ }++ const occurrences = daemonEventBatchSize*3 + 5+ var wg sync.WaitGroup+ for occurrence := range int64(occurrences) {+ for range 2 {+ wg.Go(func() { record(occurrence + 1) })+ }+ }+ wg.Wait()+ synctest.Wait()+ time.Sleep(daemonEventFlushInterval)+ synctest.Wait()+ requireDaemonCount(t, db, occurrences)+ require.Equal(t, int64(occurrences), published.Load())++ // Fewer than a batch stays queued until the flush timer, which Stop preempts.+ for occurrence := range int64(5) {+ record(occurrences + occurrence + 1)+ }+ require.NoError(t, svc.Stop(t.Context()))+ requireDaemonCount(t, db, occurrences+5)++ record(occurrences + 6)+ requireDaemonCount(t, db, occurrences+6)+ require.Equal(t, int64(occurrences+6), published.Load())+ })+}+func requireDaemonCount(t *testing.T, db *database.DB, want int64) {t.Helper()var count int64diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.goindex e36e58eeb..636d1cbec 100644--- a/backend/internal/gitops/service.go+++ b/backend/internal/gitops/service.go@@ -46,6 +46,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing")@@ -827,7 +828,7 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, idreturn nil, err}- status := &gitops.SyncStatus{+ return &gitops.SyncStatus{ID: syncRecord.ID,AutoSync: syncRecord.AutoSync,LastSyncAt: syncRecord.LastSyncAt,@@ -839,14 +840,13 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, idBackupPending: syncRecord.BackupPending,BackupFailureReason: syncRecord.BackupFailureReason,LastBackupAt: syncRecord.LastBackupAt,- }-- // Calculate next sync time- if syncRecord.AutoSync && syncRecord.LastSyncAt != nil {- status.NextSyncAt = new(syncRecord.LastSyncAt.Add(time.Duration(syncRecord.SyncInterval) * time.Minute))- }-- return status, nil+ NextSyncAt: func() *time.Time {+ if !syncRecord.AutoSync || syncRecord.LastSyncAt == nil {+ return nil+ }+ return new(syncRecord.LastSyncAt.Add(time.Duration(syncRecord.SyncInterval) * time.Minute))+ }(),+ }, nil}// CleanupLeakedScratchDirsOnStartup removes GitOps scratch directories an interrupted sync left in the projects@@ -1063,6 +1063,7 @@ func (r *backupRuntime) schedule(syncID string, run func()) {timer.Stop()}r.timers[syncID] = time.AfterFunc(r.debounce, func() {+ defer utils.RecoverToError(nil, "git backup debounce", "syncId", syncID)r.mu.Lock()delete(r.timers, syncID)r.mu.Unlock()@@ -1122,6 +1123,7 @@ func (s *GitOpsSyncService) SubscribeProjectFileChanges(ctx context.Context) {return}go func() {+ defer utils.RecoverToError(nil, "git backup after save", "syncId", syncID, "projectId", projectID)if _, err := s.PerformSync(runCtx, environmentID, syncID, user.SystemUser); err != nil {slog.ErrorContext(runCtx, "git backup after save failed", "syncId", syncID, "error", err)}diff --git a/backend/internal/health/handler.go b/backend/internal/health/handler.goindex 0e8045e5e..84014e4f5 100644--- a/backend/internal/health/handler.go+++ b/backend/internal/health/handler.go@@ -5,12 +5,11 @@ import ("net/http""github.com/getarcaneapp/arcane/types/v2/system"-- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"+ kit "go.getarcane.app/kit/pkg")type healthHandler struct {- actors *francis.Runtime+ service *Service}type healthOutput struct {@@ -22,6 +21,11 @@ type headHealthOutput struct {Status int}+type readyOutput struct {+ Status int+ Body system.ReadinessResponse+}+type healthOutputs interface {healthOutput | headHealthOutput}@@ -30,20 +34,27 @@ type healthOutputs interface {func health[T healthOutputs](h *healthHandler) func(context.Context, *struct{}) (*T, error) {return func(context.Context, *struct{}) (*T, error) {status, body := http.StatusOK, system.HealthResponse{Status: "UP"}- if h.actors != nil {- select {- case <-h.actors.Done():- status, body = http.StatusServiceUnavailable, system.HealthResponse{Status: "DOWN"}- default:- }+ select {+ case <-h.service.shutdown:+ status, body = http.StatusServiceUnavailable, system.HealthResponse{Status: "DOWN"}+ default:}var output Tswitch typed := any(&output).(type) {case *healthOutput:- typed.Status, typed.Body = status, body+ *typed = healthOutput{Status: status, Body: body}case *headHealthOutput:- typed.Status = status+ *typed = headHealthOutput{Status: status}}return &output, nil}}++// ready serves the readiness probe; 503 lists the failing check.+func (h *healthHandler) ready(ctx context.Context, _ *struct{}) (*readyOutput, error) {+ result := h.service.Ready(ctx)+ return &readyOutput{+ Status: kit.Ternary(result.Status == readyStatus, http.StatusOK, http.StatusServiceUnavailable),+ Body: result,+ }, nil+}diff --git a/backend/internal/health/module.go b/backend/internal/health/module.goindex 50fe624ef..fd216dcb3 100644--- a/backend/internal/health/module.go+++ b/backend/internal/health/module.go@@ -4,13 +4,21 @@ import ("net/http""github.com/danielgtaylor/huma/v2"-- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis")-// RegisterRoutes registers the health check routes.-func RegisterRoutes(api huma.API, actors *francis.Runtime) {- handler := &healthHandler{actors: actors}+// Module owns the liveness and readiness routes.+type Module struct {+ service *Service+}++// New builds the health module that registers the liveness and readiness routes.+func New(service *Service) *Module {+ return &Module{service: service}+}++// RegisterRoutes registers the liveness and readiness routes; readiness is hidden from the OpenAPI spec.+func (m *Module) RegisterRoutes(api huma.API) {+ handler := &healthHandler{service: m.service}huma.Register(api, huma.Operation{OperationID: "health-check",@@ -31,4 +39,15 @@ func RegisterRoutes(api huma.API, actors *francis.Runtime) {Tags: []string{"Health"},Security: []map[string][]string{},}, health[headHealthOutput](handler))++ huma.Register(api, huma.Operation{+ OperationID: "health-ready",+ Method: http.MethodGet,+ Path: "/health/ready",+ Summary: "Readiness probe",+ Description: "Report whether the database and Docker daemon are reachable",+ Tags: []string{"Health"},+ Security: []map[string][]string{},+ Hidden: true,+ }, handler.ready)}diff --git a/backend/internal/health/service.go b/backend/internal/health/service.gonew file mode 100644index 000000000..0612434d4--- /dev/null+++ b/backend/internal/health/service.go@@ -0,0 +1,82 @@+package health++import (+ "context"+ "fmt"+ "time"++ "github.com/getarcaneapp/arcane/types/v2/system"+ "github.com/moby/moby/client"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"+)++const (+ readyStatus = "ready"+ notReadyStatus = "not_ready"+ checkOK = "ok"+ checkTimeout = 2 * time.Second+)++// Service checks the dependencies the manager needs before it can serve traffic.+type Service struct {+ db *database.DB+ docker *docker.DockerClientService+ shutdown <-chan struct{}+}++// NewService builds the readiness checker over the database, the local Docker client, and the actor runtime's shutdown signal.+func NewService(db *database.DB, dockerService *docker.DockerClientService, actors *francis.Runtime) *Service {+ return &Service{+ db: db,+ docker: dockerService,+ shutdown: actors.Done(),+ }+}++// Ready pings the database and local Docker daemon; it is never ready while shutting down.+func (s *Service) Ready(ctx context.Context) system.ReadinessResponse {+ select {+ case <-s.shutdown:+ return system.ReadinessResponse{+ Status: notReadyStatus,+ Checks: map[string]string{"shutdown": "in progress"},+ }+ default:+ }++ checks := map[string]string{+ "database": func() string {+ sqlDB, err := s.db.SQLDB()+ if err != nil {+ return fmt.Sprintf("get database handle: %v", err)+ }+ pingCtx, cancel := context.WithTimeout(ctx, checkTimeout)+ defer cancel()+ if pingErr := sqlDB.PingContext(pingCtx); pingErr != nil {+ return pingErr.Error()+ }+ return checkOK+ }(),+ "docker": func() string {+ pingCtx, cancel := context.WithTimeout(ctx, checkTimeout)+ defer cancel()+ dockerClient, err := s.docker.GetClient(pingCtx)+ if err != nil {+ return fmt.Sprintf("connect to Docker: %v", err)+ }+ if _, pingErr := dockerClient.Ping(pingCtx, client.PingOptions{}); pingErr != nil {+ return pingErr.Error()+ }+ return checkOK+ }(),+ }+ for _, result := range checks {+ if result != checkOK {+ return system.ReadinessResponse{Status: notReadyStatus, Checks: checks}+ }+ }+ return system.ReadinessResponse{Status: readyStatus, Checks: checks}+}diff --git a/backend/internal/health/service_test.go b/backend/internal/health/service_test.gonew file mode 100644index 000000000..aa267ab32--- /dev/null+++ b/backend/internal/health/service_test.go@@ -0,0 +1,80 @@+package health++import (+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/getarcaneapp/arcane/types/v2/system"+ "github.com/libtnb/sqlite"+ "github.com/moby/moby/client"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "gorm.io/gorm"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+)++func newReadyTestService(t *testing.T, shutdown <-chan struct{}) (*Service, *database.DB) {+ t.Helper()++ gormDB, err := gorm.Open(sqlite.Open("file:health-ready?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)+ db := &database.DB{DB: gormDB}+ t.Cleanup(func() { _ = db.Close() })++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Api-Version", "1.41")+ _, _ = w.Write([]byte("OK"))+ }))+ t.Cleanup(server.Close)++ dockerClient, err := client.New(+ client.WithHost(server.URL),+ client.WithAPIVersion("1.41"),+ client.WithHTTPClient(server.Client()),+ )+ require.NoError(t, err)+ t.Cleanup(func() { _ = dockerClient.Close() })++ return &Service{+ db: db,+ docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),+ shutdown: shutdown,+ }, db+}++func TestService_Ready(t *testing.T) {+ t.Run("ready when database and docker respond", func(t *testing.T) {+ svc, _ := newReadyTestService(t, make(chan struct{}))++ assert.Equal(t, system.ReadinessResponse{+ Status: readyStatus,+ Checks: map[string]string{+ "database": checkOK,+ "docker": checkOK,+ },+ }, svc.Ready(t.Context()))+ })++ t.Run("not ready when database ping fails", func(t *testing.T) {+ svc, db := newReadyTestService(t, make(chan struct{}))+ require.NoError(t, db.Close())++ result := svc.Ready(t.Context())+ assert.Equal(t, notReadyStatus, result.Status)+ assert.NotEqual(t, checkOK, result.Checks["database"])+ assert.Equal(t, checkOK, result.Checks["docker"])+ })++ t.Run("not ready while shutting down", func(t *testing.T) {+ shutdown := make(chan struct{})+ close(shutdown)+ svc, _ := newReadyTestService(t, shutdown)++ result := svc.Ready(t.Context())+ assert.Equal(t, notReadyStatus, result.Status)+ assert.Equal(t, map[string]string{"shutdown": "in progress"}, result.Checks)+ })+}diff --git a/backend/internal/image/children/attestations/service.go b/backend/internal/image/children/attestations/service.goindex e5b24e218..0454853ab 100644--- a/backend/internal/image/children/attestations/service.go+++ b/backend/internal/image/children/attestations/service.go@@ -28,6 +28,7 @@ import (ocispec "github.com/opencontainers/image-spec/specs-go/v1""go.getarcane.app/kit/pkg"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth")@@ -42,8 +43,8 @@ var zstdDecoder = sync.OnceValues(func() (*zstd.Decoder, error) { return zstd.Ne// Service reads in-toto attestations attached to local images and registry references.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)- registryAuth func(ctx context.Context, registryHost string) (string, error)+ dockerService *docker.DockerClientService+ registryAuth func(ctx context.Context, registryHost string) (string, error)}type Query struct {@@ -71,10 +72,10 @@ type attestationStatementEnvelope struct {}func NewService(- dockerClient func(ctx context.Context) (*client.Client, error),+ dockerService *docker.DockerClientService,registryAuth func(ctx context.Context, registryHost string) (string, error),) *Service {- return &Service{dockerClient: dockerClient, registryAuth: registryAuth}+ return &Service{dockerService: dockerService, registryAuth: registryAuth}}// GetImageAttestations returns in-toto attestations attached to a local image or registry reference.@@ -198,28 +199,28 @@ func (s *Service) resolveImageAttestationReference(ctx context.Context, imageNamreturn reference{}, errors.New("image name is required")}- if dockerClient, err := s.dockerClient(ctx); err == nil {- if inspect, inspectErr := dockerClient.ImageInspect(ctx, imageName); inspectErr == nil {- imageRef := func() string {- for _, candidate := range slices.Concat(- slices.DeleteFunc(slices.Clone(inspect.RepoDigests), func(repoDigest string) bool { return !strings.Contains(repoDigest, "@sha256:") }),- inspect.RepoTags,- ) {- if trimmed := strings.TrimSpace(candidate); trimmed != "" && !strings.Contains(trimmed, "<none>") {- return trimmed- }+ if inspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageName)+ }); inspectErr == nil {+ imageRef := func() string {+ for _, candidate := range slices.Concat(+ slices.DeleteFunc(slices.Clone(inspect.RepoDigests), func(repoDigest string) bool { return !strings.Contains(repoDigest, "@sha256:") }),+ inspect.RepoTags,+ ) {+ if trimmed := strings.TrimSpace(candidate); trimmed != "" && !strings.Contains(trimmed, "<none>") {+ return trimmed}- return ""- }()- if imageRef == "" {- return reference{ImageRef: imageName, Empty: true}, nil}- _, subjectDigest, _ := strings.Cut(imageRef, "@")- return reference{- ImageRef: imageRef,- SubjectDigest: subjectDigest,- }, nil+ return ""+ }()+ if imageRef == "" {+ return reference{ImageRef: imageName, Empty: true}, nil}+ _, subjectDigest, _ := strings.Cut(imageRef, "@")+ return reference{+ ImageRef: imageRef,+ SubjectDigest: subjectDigest,+ }, nil}if strings.HasPrefix(imageName, "sha256:") || (len(imageName) == 64 && !strings.ContainsFunc(imageName, func(r rune) bool {diff --git a/backend/internal/image/children/patch/service.go b/backend/internal/image/children/patch/service.goindex e2b4fc69a..57e1d57e6 100644--- a/backend/internal/image/children/patch/service.go+++ b/backend/internal/image/children/patch/service.go@@ -172,14 +172,12 @@ func (s *Service) startPatch(}}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, nil, fmt.Errorf("failed to connect to Docker: %w", err)- }- if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {+ if storeErr := s.requireContainerdImageStore(ctx); storeErr != nil {return nil, nil, storeErr}- imageInspect, err := dockerClient.ImageInspect(ctx, imageID)+ imageInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageID)+ })if err != nil {return nil, nil, fmt.Errorf("failed to inspect image: %w", err)}@@ -435,15 +433,15 @@ func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts i// verifyPatchedImage warns when the expected tag is missing (copa may arch-suffix// multi-platform tags) and otherwise re-scans it to show whether the patch worked.func (s *Service) verifyPatchedImage(ctx context.Context, record *ImagePatchRecord, activityID string) {- if dockerClient, err := s.dockerService.GetClient(ctx); err == nil {- if patchedInspect, inspectErr := dockerClient.ImageInspect(ctx, record.PatchedRef); inspectErr != nil {- slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", inspectErr)- s.appendPatchActivity(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")- } else if s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {- s.appendPatchActivity(ctx, activityID, 95, "Re-scanning patched image to verify the patch")- if _, scanErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, scheduledPatchUser); scanErr != nil {- slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanErr)- }+ if patchedInspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, record.PatchedRef)+ }); inspectErr != nil {+ slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", inspectErr)+ s.appendPatchActivity(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")+ } else if s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ s.appendPatchActivity(ctx, activityID, 95, "Re-scanning patched image to verify the patch")+ if _, scanErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, scheduledPatchUser); scanErr != nil {+ slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanErr)}}}@@ -844,11 +842,7 @@ func (s *Service) Targets(ctx context.Context, envID string) ([]Target, error) {if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {return nil, err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }- if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {+ if storeErr := s.requireContainerdImageStore(ctx); storeErr != nil {return nil, storeErr}var scans []vulnerability.VulnerabilityScanRecord@@ -885,8 +879,10 @@ func (s *Service) Targets(ctx context.Context, envID string) ([]Target, error) {}// Copa needs BuildKit's docker exporter, which dockerd only offers with the containerd image store.-func requireContainerdImageStore(ctx context.Context, dockerClient *client.Client) error {- info, err := dockerClient.Info(ctx, client.InfoOptions{})+func (s *Service) requireContainerdImageStore(ctx context.Context) error {+ info, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SystemInfoResult, error) {+ return dockerClient.Info(ctx, client.InfoOptions{})+ })if err != nil {return fmt.Errorf("failed to inspect Docker: %w", err)}diff --git a/backend/internal/image/module.go b/backend/internal/image/module.goindex b6c2c5497..5ea746a5b 100644--- a/backend/internal/image/module.go+++ b/backend/internal/image/module.go@@ -42,15 +42,14 @@ func New(uploadService *upload.UploadService,) *Module {return &Module{- service: service,- docker: dockerService,- imageUpdate: imageUpdate,- settings: settingsService,- build: buildService,- activity: activityService,- upload: uploadService,- // The attestation reader uses the image service's Docker client and registry credentials.- attestationsHandler: attestations.NewHandler(attestations.NewService(service.dockerService.GetClient, service.registryService.GetRegistryAuthForHost)),+ service: service,+ docker: dockerService,+ imageUpdate: imageUpdate,+ settings: settingsService,+ build: buildService,+ activity: activityService,+ upload: uploadService,+ attestationsHandler: attestations.NewHandler(attestations.NewService(service.dockerService, service.registryService.GetRegistryAuthForHost)),}}diff --git a/backend/internal/image/service.go b/backend/internal/image/service.goindex 9980a94ed..a74c79679 100644--- a/backend/internal/image/service.go+++ b/backend/internal/image/service.go@@ -47,6 +47,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"@@ -225,11 +226,6 @@ func (s *ImageService) PatchWorkflow() *flow.Workflow { return s.patchWorkflow }// and ImageList concurrently so the size field reflects the same metric shown in the// image table (docker image ls / docker system df).func (s *ImageService) GetImageDetail(ctx context.Context, id string) (*imagetypes.DetailSummary, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-var (inspect image.InspectResponselistSize int64@@ -241,8 +237,9 @@ func (s *ImageService) GetImageDetail(ctx context.Context, id string) (*imagetypg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "image worker")- var inspectImageErr error- inspectResult, inspectImageErr := dockerClient.ImageInspect(gctx, id)+ inspectResult, inspectImageErr := s.dockerService.Call(gctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, id)+ })if inspectImageErr != nil {return fmt.Errorf("inspect not found: %w", inspectImageErr)}@@ -294,13 +291,9 @@ func (s *ImageService) GetImageDetail(ctx context.Context, id string) (*imagetyp}func (s *ImageService) RemoveImage(ctx context.Context, id string, force bool, user usertypes.Actor) error {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", id, "", user.ID, user.Username, "0", err, database.JSON{"action": "delete", "force": force})- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- imageDetails, inspectErr := dockerClient.ImageInspect(ctx, id)+ imageDetails, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, id)+ })imageName := func() string {if inspectErr == nil && len(imageDetails.RepoTags) > 0 {return imageDetails.RepoTags[0]@@ -321,21 +314,18 @@ func (s *ImageService) RemoveImage(ctx context.Context, id string, force bool, utagReleases = append(tagReleases, s.eventService.BeginDockerResourceSuppressionWindow("image", "", tag))}}- removed, err := dockerClient.ImageRemove(ctx, id, client.ImageRemoveOptions{- Force: force,- PruneChildren: true,+ removed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageRemoveResult, error) {+ return dockerClient.ImageRemove(ctx, id, client.ImageRemoveOptions{+ Force: force,+ PruneChildren: true,+ })})if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", id, imageName, user.ID, user.Username, "0", err, database.JSON{"action": "delete", "force": force})return fmt.Errorf("failed to remove image: %w", err)}- s.cleanupImageUpdateRecords(ctx, append(getDeletedImageIDs(removed.Items), id))-- // Clean up vulnerability scan records for the deleted image- if deleteScanResultErr := s.vulnerabilityService.DeleteScanResult(ctx, id); deleteScanResultErr != nil {- slog.WarnContext(ctx, "failed to delete vulnerability scan record", "id", id, "error", deleteScanResultErr)- }+ s.cleanupRemovedImages(ctx, removed.Items, id)if logErr := s.eventService.LogImageEvent(ctx, event.EventTypeImageDelete, id, imageName, user.ID, user.Username, "0", database.JSON{"action": "delete",@@ -348,6 +338,26 @@ func (s *ImageService) RemoveImage(ctx context.Context, id string, force bool, ureturn nil}+// cleanupRemovedImages drops update and vulnerability records for images Docker deleted or untagged.+func (s *ImageService) cleanupRemovedImages(ctx context.Context, deleted []image.DeleteResponse, extraIDs ...string) {+ idsToDelete := make([]string, 0, len(deleted)+len(extraIDs))+ for _, img := range deleted {+ if deletedID := cmp.Or(img.Deleted, img.Untagged); deletedID != "" {+ idsToDelete = append(idsToDelete, deletedID)+ }+ }+ idsToDelete = append(idsToDelete, extraIDs...)+ if recordsErr := s.imageUpdateService.DeleteRecordsForImages(ctx, idsToDelete); recordsErr != nil {+ slog.WarnContext(ctx, "failed to clean up image update records", "error", recordsErr)+ }+ if orphanErr := s.imageUpdateService.CleanupOrphanedRecords(ctx); orphanErr != nil {+ slog.WarnContext(ctx, "failed to clean up orphaned image update records", "error", orphanErr)+ }+ if deleteErr := s.vulnerabilityService.DeleteScanResultsByImageIDs(ctx, idsToDelete); deleteErr != nil {+ slog.WarnContext(ctx, "failed to delete vulnerability scan records", "count", len(idsToDelete), "error", deleteErr)+ }+}+func (s *ImageService) PullImage(ctx context.Context, imageName string, progressWriter io.Writer, user usertypes.Actor, externalCreds []containerregistry.Credential) error {dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {@@ -428,24 +438,69 @@ func (s *ImageService) PullImage(ctx context.Context, imageName string, progress// engine doesn't report it. Pull-policy refresh windows compare against this// timestamp, the same clock compose v5.5.0 uses.func (s *ImageService) ImageLastTagTime(ctx context.Context, imageName string) (time.Time, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return time.Time{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- inspect, err := dockerClient.ImageInspect(ctx, imageName)+ inspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageName)+ })if err != nil {return time.Time{}, fmt.Errorf("failed to inspect image %s: %w", imageName, err)}return inspect.Metadata.LastTagTime, nil}-func (s *ImageService) ReconcilePulledImageUpdate(ctx context.Context, imageName string) error {- if s.imageUpdateService == nil {- return nil+// PullRequest describes one pull through Pull.+type PullRequest struct {+ // Ref is the reference pulled from the registry.+ Ref string+ // Tag, when set, is the reference Ref is tagged as after the pull; it is the one reconciled.+ Tag string+ Progress io.Writer+ User usertypes.Actor+ Credentials []containerregistry.Credential+}++// Pull pulls req.Ref within the dockerImagePullTimeout setting and reconciles the pulled reference's update state.+func (s *ImageService) Pull(ctx context.Context, req PullRequest) error {+ pullCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))+ defer cancel()++ pull := func(ctx context.Context) error {+ return s.PullImage(ctx, req.Ref, req.Progress, req.User, req.Credentials)}+ pullErr := func() error {+ if req.Tag == "" || req.Tag == req.Ref {+ return pull(pullCtx)+ }+ // A digest pull is tagged back to the reference the caller selected.+ dockerClient, clientErr := s.dockerService.GetClient(pullCtx)+ if clientErr != nil {+ return clientErr+ }+ return imageref.PullAsTag(pullCtx, dockerClient, req.Ref, req.Tag, pull)+ }()+ if pullErr != nil {+ if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {+ return fmt.Errorf("image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting): %w", req.Ref, context.DeadlineExceeded)+ }+ return fmt.Errorf("failed to pull image %s: %w", req.Ref, pullErr)+ }++ // Reconcile on the caller's context so a late pull still clears its update records (#4306).+ if reconcileErr := s.imageUpdateService.MarkImageRefUpToDateAfterPull(ctx, cmp.Or(req.Tag, req.Ref)); reconcileErr != nil {+ slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", cmp.Or(req.Tag, req.Ref), "error", reconcileErr)+ }+ return nil+}++// RemoteDigest resolves imageRef's registry digest within the registry timeout setting.+func (s *ImageService) RemoteDigest(ctx context.Context, imageRef string, credentials []containerregistry.Credential) (string, error) {+ digestCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().RegistryTimeout.AsInt(), timeouts.DefaultRegistry))+ defer cancel()- return s.imageUpdateService.MarkImageRefUpToDateAfterPull(ctx, imageName)+ result, err := s.registryService.InspectImageDigest(digestCtx, imageRef, credentials)+ if err != nil {+ return "", err+ }+ return kit.FromPtr(result).Digest, nil}// TagImage adds a repository tag to an existing image.@@ -462,14 +517,10 @@ func (s *ImageService) TagImage(ctx context.Context, source string, req imagetyptarget := kit.Ternary(tag != "", repository+":"+tag, repository)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", "", source, user.ID, user.Username, "0", err, database.JSON{"action": "tag", "target": target})- return fmt.Errorf("failed to connect to Docker: %w", err)- }-defer s.eventService.BeginDockerResourceSuppressionWindow("image", "", target)()- _, err = dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: source, Target: target})+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageTagResult, error) {+ return dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: source, Target: target})+ })if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", "", source, user.ID, user.Username, "0", err, database.JSON{"action": "tag", "target": target})return fmt.Errorf("failed to tag image: %w", err)@@ -495,12 +546,9 @@ func (s *ImageService) GetImageHistory(ctx context.Context, imageName string) ([return nil, errors.New("image name is required")}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- result, err := dockerClient.ImageHistory(ctx, imageName)+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageHistoryResult, error) {+ return dockerClient.ImageHistory(ctx, imageName)+ })if err != nil {return nil, fmt.Errorf("failed to get image history: %w", err)}@@ -526,12 +574,9 @@ func (s *ImageService) SearchImages(ctx context.Context, term string) ([]imagetyreturn nil, errors.New("search term is required")}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- result, err := dockerClient.ImageSearch(ctx, term, client.ImageSearchOptions{})+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageSearchResult, error) {+ return dockerClient.ImageSearch(ctx, term, client.ImageSearchOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to search images: %w", err)}@@ -622,12 +667,9 @@ func (s *ImageService) LoadImageFromReader(ctx context.Context, reader io.Reader}func (s *ImageService) ImageExistsLocally(ctx context.Context, imageName string) (bool, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return false, fmt.Errorf("failed to connect to Docker: %w", err)- }-- _, err = dockerClient.ImageInspect(ctx, imageName)+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageName)+ })if err == nil {return true, nil}@@ -684,11 +726,6 @@ func ShouldRetryAnonymousPull(pullOptions client.ImagePullOptions, pullErr error}func (s *ImageService) PruneImages(ctx context.Context, options system.PruneImagesOptions) (*image.PruneReport, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-filterArgs, err := func() (client.Filters, error) {switch options.Mode {case system.PruneImageModeNone:@@ -710,18 +747,16 @@ func (s *ImageService) PruneImages(ctx context.Context, options system.PruneImagreturn nil, err}+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to prune images: %w", err)+ }report, err := dockerClient.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})if err != nil {return nil, fmt.Errorf("failed to prune images: %w", err)}- idsToDelete := getDeletedImageIDs(report.Report.ImagesDeleted)- s.cleanupImageUpdateRecords(ctx, idsToDelete)- if len(idsToDelete) > 0 {- if deleteErr := s.vulnerabilityService.DeleteScanResultsByImageIDs(ctx, idsToDelete); deleteErr != nil {- slog.WarnContext(ctx, "failed to delete vulnerability scan records after prune", "count", len(idsToDelete), "error", deleteErr)- }- }+ s.cleanupRemovedImages(ctx, report.Report.ImagesDeleted)if logErr := s.eventService.LogImageEvent(ctx, event.EventTypeImageDelete, "", "bulk_prune", usertypes.SystemUser.ID, usertypes.SystemUser.Username, "0", database.JSON{"action": "prune",@@ -736,33 +771,6 @@ func (s *ImageService) PruneImages(ctx context.Context, options system.PruneImagreturn &report.Report, nil}-func getDeletedImageIDs(deletedImages []image.DeleteResponse) []string {- if len(deletedImages) == 0 {- return nil- }-- idsToDelete := make([]string, 0, len(deletedImages))- for _, img := range deletedImages {- if img.Deleted != "" {- idsToDelete = append(idsToDelete, img.Deleted)- continue- }- if img.Untagged != "" {- idsToDelete = append(idsToDelete, img.Untagged)- }- }- return idsToDelete-}--func (s *ImageService) cleanupImageUpdateRecords(ctx context.Context, idsToDelete []string) {- if err := s.imageUpdateService.DeleteRecordsForImages(ctx, idsToDelete); err != nil {- slog.WarnContext(ctx, "failed to clean up image update records", "error", err)- }- if err := s.imageUpdateService.CleanupOrphanedRecords(ctx); err != nil {- slog.WarnContext(ctx, "failed to clean up orphaned image update records", "error", err)- }-}-// GetUpdateInfoByImageIDs returns a map of image ID to UpdateInfo for the given image IDs.// This is used by the container service to populate update info for containers.func (s *ImageService) GetUpdateInfoByImageIDs(ctx context.Context, imageIDs []string) (map[string]*imagetypes.UpdateInfo, error) {@@ -813,16 +821,7 @@ func (s *ImageService) GetUpdateInfoByContainers(ctx context.Context, containers}if len(digestImageIDs) > 0 {- imageIDs := make([]string, 0, len(digestImageIDs))- seen := make(map[string]struct{}, len(digestImageIDs))- for _, imageID := range digestImageIDs {- if _, exists := seen[imageID]; exists {- continue- }- seen[imageID] = struct{}{}- imageIDs = append(imageIDs, imageID)- }- byImageID, err := s.GetUpdateInfoByImageIDs(ctx, imageIDs)+ byImageID, err := s.GetUpdateInfoByImageIDs(ctx, kit.Unique(slices.Collect(maps.Values(digestImageIDs))))if err != nil {return nil, err}@@ -879,23 +878,12 @@ func (s *ImageService) GetUpdateInfoByImageRefs(ctx context.Context, imageRefs [}repositoryCandidates := make([]string, 0)- repositorySeen := make(map[string]struct{})tags := make([]string, 0, len(lookups))- tagSeen := make(map[string]struct{})-for _, lookup := range lookups {- if _, exists := tagSeen[lookup.tag]; !exists {- tagSeen[lookup.tag] = struct{}{}- tags = append(tags, lookup.tag)- }- for repo := range lookup.repositoryCandidates {- if _, exists := repositorySeen[repo]; exists {- continue- }- repositorySeen[repo] = struct{}{}- repositoryCandidates = append(repositoryCandidates, repo)- }+ tags = append(tags, lookup.tag)+ repositoryCandidates = slices.AppendSeq(repositoryCandidates, maps.Keys(lookup.repositoryCandidates))}+ repositoryCandidates, tags = kit.Unique(repositoryCandidates), kit.Unique(tags)if len(repositoryCandidates) == 0 || len(tags) == 0 {return result, nil@@ -913,7 +901,10 @@ func (s *ImageService) GetUpdateInfoByImageRefs(ctx context.Context, imageRefs [// earliest position, matching the order the records were fetched in.index := make(map[imageUpdateRecordKey]int, len(updateRecords))for i := range updateRecords {- key := newImageUpdateRecordKey(updateRecords[i].Repository, updateRecords[i].Tag)+ key := imageUpdateRecordKey{+ repository: strings.TrimSpace(updateRecords[i].Repository),+ tag: strings.ToLower(strings.TrimSpace(updateRecords[i].Tag)),+ }if existing, ok := index[key]; !ok || updateRecords[i].CheckTime.After(updateRecords[existing].CheckTime) {index[key] = i}@@ -922,7 +913,7 @@ func (s *ImageService) GetUpdateInfoByImageRefs(ctx context.Context, imageRefs [// Pick the latest record across the lookup's repository aliases; equal check times fall back to fetch order.latest := -1for repository := range lookup.repositoryCandidates {- i, ok := index[newImageUpdateRecordKey(repository, lookup.tag)]+ i, ok := index[imageUpdateRecordKey{repository: strings.TrimSpace(repository), tag: strings.ToLower(strings.TrimSpace(lookup.tag))}]if !ok {continue}@@ -1066,10 +1057,6 @@ func (s *ImageService) AggregateImageCounts(ctx context.Context) (base.ApiRespon}, nil}-func newImageUpdateRecordKey(repository, tag string) imageUpdateRecordKey {- return imageUpdateRecordKey{repository: strings.TrimSpace(repository), tag: strings.ToLower(strings.TrimSpace(tag))}-}-// BuildProjectIDMap returns a map of compose project name → project ID for any// containers that carry the com.docker.compose.project label. The lookup uses a// short-TTL cache shared across all callers of this ImageService instance.@@ -1248,10 +1235,8 @@ func MapDockerImagesToDTOs(}return di.RepoTags[0], "latest"}- for _, rd := range di.RepoDigests {- if candidateRepo, _, found := strings.CutLast(rd, "@"); rd != "<none>@<none>" && found && candidateRepo != "" {- return candidateRepo, "<none>"- }+ if candidateRepo, found := imageref.RepoDigestRepository(di.RepoDigests); found {+ return candidateRepo, "<none>"}return "<none>", "<none>"}()diff --git a/backend/internal/imageupdate/service.go b/backend/internal/imageupdate/service.goindex aadfa84c2..07aadb249 100644--- a/backend/internal/imageupdate/service.go+++ b/backend/internal/imageupdate/service.go@@ -171,13 +171,6 @@ func NewImageUpdateService(}}-func (s *ImageUpdateService) dockerAPIContext(ctx context.Context) (context.Context, context.CancelFunc) {- if s.settingsService == nil {- return context.WithTimeout(ctx, timeouts.DefaultDockerAPI)- }- return context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))-}-func (s *ImageUpdateService) registryContext(ctx context.Context) (context.Context, context.CancelFunc) {if s.settingsService == nil {return context.WithTimeout(ctx, timeouts.DefaultRegistry)@@ -185,19 +178,7 @@ func (s *ImageUpdateService) registryContext(ctx context.Context) (context.Contereturn context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().RegistryTimeout.AsInt(), timeouts.DefaultRegistry))}-func (s *ImageUpdateService) dockerClient(ctx context.Context) (*client.Client, error) {- if s.dockerService == nil {- return nil, errors.New("docker service unavailable")- }- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }- return dockerClient, nil-}-func (s *ImageUpdateService) composeBuildImageRefs(ctx context.Context) (map[string]struct{}, error) {- buildRefs := make(map[string]struct{})var projectRows []struct {ID stringBuildImageRefsJSON *string `gorm:"column:build_image_refs_json"`@@ -210,15 +191,6 @@ func (s *ImageUpdateService) composeBuildImageRefs(ctx context.Context) (map[strreturn nil, fmt.Errorf("load project build image references: %w", err)}- for _, row := range projectRows {- for _, imageRef := range projects.ParseImageRefsJSON(kit.FromPtr(row.BuildImageRefsJSON)) {- normalized := refs.NormalizeImageUpdateRef(imageRef)- if normalized != "" {- buildRefs[normalized] = struct{}{}- }- }- }-// Copacetic-patched tags only exist in the local daemon, so treat them like// locally built images instead of failing a remote registry lookup. Never// fail the update check over patch history.@@ -230,9 +202,13 @@ func (s *ImageUpdateService) composeBuildImageRefs(ctx context.Context) (map[strPluck("patched_ref", &patchedRefs).Error; err != nil {slog.DebugContext(ctx, "failed to load patched image references", "error", err)}+ for _, row := range projectRows {+ patchedRefs = append(patchedRefs, projects.ParseImageRefsJSON(kit.FromPtr(row.BuildImageRefsJSON))...)+ }++ buildRefs := make(map[string]struct{}, len(patchedRefs))for _, imageRef := range patchedRefs {- normalized := refs.NormalizeImageUpdateRef(imageRef)- if normalized != "" {+ if normalized := refs.NormalizeImageUpdateRef(imageRef); normalized != "" {buildRefs[normalized] = struct{}{}}}@@ -404,7 +380,13 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef striif kit.FromPtr(snapshot).ImageID != "" {return snapshot.ImageID, nil}- return s.getImageIDByRef(ctx, imageRef)+ inspectResponse, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageRef)+ })+ if inspectErr != nil {+ return "", fmt.Errorf("image not found: %w", inspectErr)+ }+ return inspectResponse.ID, nil}()switch {case imageIDErr != nil:@@ -533,18 +515,12 @@ func (s *ImageUpdateService) parseImageReference(imageRef string) *ImageParts {}func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string) (string, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return "", err- }-imageID = strings.TrimPrefix(imageID, "sha256:")if imageRef, ok := func() (string, bool) {- apiCtx, cancel := s.dockerAPIContext(ctx)- defer cancel()-- inspectResponse, inspectErr := dockerClient.ImageInspect(apiCtx, imageID)+ inspectResponse, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageID)+ })if inspectErr != nil {return "", false}@@ -556,12 +532,8 @@ func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string}}}- for _, digestValue := range inspectResponse.RepoDigests {- if digestValue != "<none>@<none>" {- if repo, _, found := strings.Cut(digestValue, "@"); found {- return repo + ":latest", true- }- }+ if repo, found := imageref.RepoDigestRepository(inspectResponse.RepoDigests); found {+ return repo + ":latest", true}return "", false}(); ok {@@ -571,10 +543,9 @@ func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string// Fallback: if the image was pruned, look up the image reference from// running containers that were started from this image ID.if imageRef, ok := func() (string, bool) {- apiCtx, cancel := s.dockerAPIContext(ctx)- defer cancel()-- containers, listErr := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})+ containers, listErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if listErr != nil {return "", false}@@ -595,21 +566,15 @@ func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string}func (s *ImageUpdateService) getAllImageRefs(ctx context.Context, limit int) ([]string, error) {- imageCtx, cancelImage := s.dockerAPIContext(ctx)- images, err := s.dockerService.ListImages(imageCtx)- cancelImage()+ images, err := s.dockerService.ListImages(ctx)if err != nil {return nil, fmt.Errorf("failed to list Docker images: %w", err)}- containerCtx, cancelContainers := s.dockerAPIContext(ctx)- containers, err := s.dockerService.ListContainers(containerCtx)- cancelContainers()+ containers, err := s.dockerService.ListContainers(ctx)if err != nil {slog.WarnContext(ctx, "failed to list Docker containers; continuing without update-check opt-out filtering", "error", err.Error())- return filterImageSummariesByContainerOptOut(images, nil, limit), nil}-return filterImageSummariesByContainerOptOut(images, containers, limit), nil}@@ -695,15 +660,9 @@ func filterImageSummariesByContainerOptOut(images []image.Summary, containers []}func (s *ImageUpdateService) inspectLocalImageSnapshot(ctx context.Context, imageRef string, composeBuildRefs map[string]struct{}) (*localImageSnapshot, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, err- }-- apiCtx, cancel := s.dockerAPIContext(ctx)- defer cancel()-- inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageRef)+ inspectResponse, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageRef)+ })if err != nil {return nil, fmt.Errorf("failed to inspect image: %w", err)}@@ -725,12 +684,12 @@ func (s *ImageUpdateService) inspectLocalImageSnapshot(ctx context.Context, imag}}- repo, tag := extractRepoAndTagFromImage(inspectResponse.InspectResponse)+ repo, tag := extractRepoAndTagFromImage(inspectResponse.InspectResponse, s.parseImageReference(imageRef))return &localImageSnapshot{ImageID: inspectResponse.ID,Repository: repo,- Tag: tagWithFallback(tag, s.parseImageReference(imageRef)),+ Tag: tag,// Use the first repository digest, else the first digest; fallback to image ID if no repo digests availablePrimaryDigest: slices.Concat(repositoryDigests, allDigests, []string{inspectResponse.ID})[0],AllDigests: kit.Ternary(len(allDigests) > 0, allDigests, []string{inspectResponse.ID}),@@ -756,7 +715,17 @@ func (s *ImageUpdateService) CheckImageUpdateByID(ctx context.Context, imageID sreturn nil, err}if len(result.ContainerUpdates) == 0 {- if saveErr := s.saveUpdateResultByID(ctx, imageID, result, s.parseImageReference(imageRef)); saveErr != nil {+ saveErr := func() error {+ dockerImage, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageID)+ })+ if inspectErr != nil {+ return fmt.Errorf("failed to inspect image: %w", inspectErr)+ }+ repo, tag := extractRepoAndTagFromImage(dockerImage.InspectResponse, s.parseImageReference(imageRef))+ return s.savePreparedUpdateResult(ctx, imageID, repo, tag, result)+ }()+ if saveErr != nil {slog.WarnContext(ctx, "Failed to save update result by ID", "imageId", imageID, "error", saveErr.Error())}}@@ -772,7 +741,9 @@ func (s *ImageUpdateService) saveUpdateResultWithSnapshot(ctx context.Context, iif parts == nil {return errors.New("invalid image reference")}- imageID, err := s.getImageIDByRef(ctx, imageRef)+ inspectResponse, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, imageRef)+ })if err != nil {repository := func() string {repositoryPath := strings.TrimSpace(parts.Repository)@@ -796,10 +767,13 @@ func (s *ImageUpdateService) saveUpdateResultWithSnapshot(ctx context.Context, ireturn s.savePreparedUpdateResult(ctx, syntheticID, repository, parts.Tag, result)}- return s.saveUpdateResultByID(ctx, imageID, result, parts)+ repo, tag := extractRepoAndTagFromImage(inspectResponse.InspectResponse, parts)+ return s.savePreparedUpdateResult(ctx, inspectResponse.ID, repo, tag, result)}-func extractRepoAndTagFromImage(dockerImage image.InspectResponse) (repo, tag string) {+// extractRepoAndTagFromImage reads the image's first usable tag; an untagged image uses fallback's tag when set.+func extractRepoAndTagFromImage(dockerImage image.InspectResponse, fallback *ImageParts) (repo, tag string) {+ untagged := cmp.Or(strings.TrimSpace(kit.FromPtr(fallback).Tag), "<none>")if len(dockerImage.RepoTags) > 0 && dockerImage.RepoTags[0] != "<none>:<none>" {if named, err := reference.ParseNormalizedNamed(dockerImage.RepoTags[0]); err == nil {if tagged, ok := named.(reference.NamedTagged); ok {@@ -815,16 +789,11 @@ func extractRepoAndTagFromImage(dockerImage image.InspectResponse) (repo, tag streturn parts[0], "latest"}- for _, rd := range dockerImage.RepoDigests {- if rd == "<none>@<none>" {- continue- }- if repoCandidate, _, found := strings.CutLast(rd, "@"); found && repoCandidate != "" {- return repoCandidate, "<none>"- }+ if repoCandidate, found := imageref.RepoDigestRepository(dockerImage.RepoDigests); found {+ return repoCandidate, untagged}- return "<none>", "<none>"+ return "<none>", untagged}func buildImageUpdateRecord(imageID, repo, tag string, result *imageupdate.Response, notificationSent bool, latestSeenAt *time.Time, carried *ImageUpdateRecord) *ImageUpdateRecord {@@ -904,53 +873,12 @@ func savePreparedUpdateResultWithTx(tx *gorm.DB, imageID, repo, tag string, resu)).Error}-func (s *ImageUpdateService) saveUpdateResultByID(ctx context.Context, imageID string, result *imageupdate.Response, fallback *ImageParts) error {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return err- }-- apiCtx, cancel := s.dockerAPIContext(ctx)- defer cancel()-- dockerImage, err := dockerClient.ImageInspect(apiCtx, imageID)- if err != nil {- return fmt.Errorf("failed to inspect image: %w", err)- }-- repo, tag := extractRepoAndTagFromImage(dockerImage.InspectResponse)- return s.savePreparedUpdateResult(ctx, imageID, repo, tagWithFallback(tag, fallback), result)-}--func tagWithFallback(tag string, fallback *ImageParts) string {- if tag == "<none>" && strings.TrimSpace(kit.FromPtr(fallback).Tag) != "" {- return fallback.Tag- }- return tag-}-func (s *ImageUpdateService) savePreparedUpdateResult(ctx context.Context, imageID, repo, tag string, result *imageupdate.Response) error {return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {return savePreparedUpdateResultWithTx(tx, imageID, repo, tag, result)})}-func (s *ImageUpdateService) getImageIDByRef(ctx context.Context, imageRef string) (string, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return "", err- }-- apiCtx, cancel := s.dockerAPIContext(ctx)- defer cancel()-- inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageRef)- if err != nil {- return "", fmt.Errorf("image not found: %w", err)- }- return inspectResponse.ID, nil-}-func (s *ImageUpdateService) MarkImageRefUpToDateAfterPull(ctx context.Context, imageRef string) error {snapshot, err := s.inspectLocalImageSnapshot(ctx, imageRef, nil)if err != nil {@@ -1404,13 +1332,9 @@ func (s *ImageUpdateService) CheckImages(ctx context.Context, request imageupdat// MonitoredImageRefs keeps the refs whose containers still permit update checks.func (s *ImageUpdateService) MonitoredImageRefs(ctx context.Context, imageRefs []string) ([]string, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, err- }- apiCtx, cancel := s.dockerAPIContext(ctx)- listed, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})- cancel()+ listed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if err != nil {return nil, fmt.Errorf("list containers for update-check eligibility: %w", err)}@@ -1683,13 +1607,9 @@ func (s *ImageUpdateService) SendBatchUpdateNotifications(ctx context.Context) e// Records for resources that opted out of monitoring since the check, or// whose container is gone, stay unnotified: they resurface if monitoring// is re-enabled and are otherwise removed by orphan cleanup.- dockerClient, clientErr := s.dockerClient(notifCtx)- if clientErr != nil {- return records, fmt.Errorf("resolve update-check eligibility: %w", clientErr)- }- apiCtx, apiCancel := s.dockerAPIContext(notifCtx)- listed, listErr := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})- apiCancel()+ listed, listErr := s.dockerService.Call(notifCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if listErr != nil {return records, fmt.Errorf("list containers for update-check eligibility: %w", listErr)}@@ -1869,13 +1789,9 @@ func (s *ImageUpdateService) checkContainerTagUpdates(ctx context.Context, imageif len(wanted) == 0 {return map[string]*imageupdate.Response{}, nil}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return map[string]*imageupdate.Response{}, err- }- apiCtx, cancel := s.dockerAPIContext(ctx)- listed, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})- cancel()+ listed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if err != nil {return map[string]*imageupdate.Response{}, fmt.Errorf("list containers for tag update checks: %w", err)}diff --git a/backend/internal/imageupdate/service_test.go b/backend/internal/imageupdate/service_test.goindex 8619c3bca..7894e8f60 100644--- a/backend/internal/imageupdate/service_test.go+++ b/backend/internal/imageupdate/service_test.go@@ -781,7 +781,7 @@ func TestImageUpdateService_CheckImageUpdate_SkipsDigestPinnedReference(t *testipinnedDigest := digest.FromString("pinned-newt").String()imageRef := "ghcr.io/fosrl/newt@" + pinnedDigesteventService := event.NewEventService(db, nil, nil)- svc := NewImageUpdateService(db, nil, nil, nil, eventService, notification.NewNotificationService(db, nil, nil, nil, nil), nil)+ svc := NewImageUpdateService(db, nil, nil, newImageUpdateNotificationDockerService(t, nil), eventService, notification.NewNotificationService(db, nil, nil, nil, nil), nil)result, err := svc.CheckImageUpdate(t.Context(), imageRef)require.NoError(t, err)@@ -797,7 +797,7 @@ func TestImageUpdateService_CheckImages_SkipsDigestPinnedReference(t *testing.T)db := setupImageUpdateTestDB(t)pinnedDigest := digest.FromString("batch-pinned-newt").String()imageRef := "ghcr.io/fosrl/newt@" + pinnedDigest- svc := NewImageUpdateService(db, nil, nil, nil, nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)+ svc := NewImageUpdateService(db, nil, nil, newImageUpdateNotificationDockerService(t, nil), nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)require.Equal(t, [][]string{{imageRef}}, svc.groupImageRefs([]string{imageRef}))@@ -828,7 +828,7 @@ func TestImageUpdateService_CheckImages_SkippedDigestPinnedReferenceClearsStaleELastError: new("failed to get remote digest"),CheckTime: time.Now().Add(-time.Hour),}).Error)- svc := NewImageUpdateService(db, nil, nil, nil, nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)+ svc := NewImageUpdateService(db, nil, nil, newImageUpdateNotificationDockerService(t, nil), nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)results, err := checkImages(t, t.Context(), svc, imageRef)require.NoError(t, err)@@ -1126,7 +1126,7 @@ func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeout(t *testing.T) {db,settingsService,nil,- &docker.DockerClientService{Client: newImageUpdateTestDockerClient(t, server)},+ docker.NewDockerClientService(t.Context(), nil, nil, settingsService).WithClient(newImageUpdateTestDockerClient(t, server)),nil,notification.NewNotificationService(db, nil, nil, nil, nil),nil,@@ -1152,7 +1152,7 @@ func TestImageUpdateService_InspectLocalImageSnapshotUsesDockerAPITimeout(t *tesdb,settingsService,nil,- &docker.DockerClientService{Client: newImageUpdateTestDockerClient(t, server)},+ docker.NewDockerClientService(t.Context(), nil, nil, settingsService).WithClient(newImageUpdateTestDockerClient(t, server)),nil,notification.NewNotificationService(db, nil, nil, nil, nil),nil,@@ -1962,10 +1962,14 @@ func TestImageUpdateService_SendBatchNotifications_UnresolvedEligibilityLeavesUncalls := newImageUpdateGenericWebhookProvider(t, db)notif := notification.NewNotificationService(db, nil, nil, nil, nil)- svc := NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ }))+ t.Cleanup(server.Close)+ svc := NewImageUpdateService(db, nil, nil, &docker.DockerClientService{Client: newImageUpdateTestDockerClient(t, server)}, nil, notif, nil)require.NoError(t, db.Create(&ImageUpdateRecord{ID: "sha256:img", Repository: "test/repo", Tag: "latest", HasUpdate: true}).Error)- require.ErrorContains(t, svc.SendBatchUpdateNotifications(t.Context()), "resolve update-check eligibility: docker service unavailable")+ require.ErrorContains(t, svc.SendBatchUpdateNotifications(t.Context()), "list containers for update-check eligibility")require.Zero(t, calls.Load(), "nothing is sent when eligibility cannot be resolved")var reloaded ImageUpdateRecorddiff --git a/backend/internal/network/service.go b/backend/internal/network/service.goindex 2708831e7..b3c581d8e 100644--- a/backend/internal/network/service.go+++ b/backend/internal/network/service.go@@ -69,12 +69,9 @@ func NewNetworkService(db *database.DB, dockerService *dockerInternal.DockerClie}func (s *NetworkService) GetNetworkByID(ctx context.Context, id string) (*network.Inspect, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- networkInspect, err := compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})+ networkInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {+ return compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("network not found: %w", err)}@@ -122,11 +119,6 @@ func (s *NetworkService) GetNetworkTopology(ctx context.Context) (*networktypes.}func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topology, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-containers, err := s.dockerService.ListContainers(ctx)if err != nil {return nil, fmt.Errorf("failed to list containers: %w", err)@@ -137,7 +129,9 @@ func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.TopolcontainersByID[rawContainer.ID] = rawContainer}- networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ networkList, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkListResult, error) {+ return compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list Docker networks: %w", err)}@@ -151,7 +145,9 @@ func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topolg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "network worker")- inspected, inspectErr := compat.NetworkInspectWithCompatibility(groupCtx, dockerClient, rawNetwork.ID, client.NetworkInspectOptions{})+ inspected, inspectErr := s.dockerService.Call(groupCtx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {+ return compat.NetworkInspectWithCompatibility(ctx, dockerClient, rawNetwork.ID, client.NetworkInspectOptions{})+ })if inspectErr != nil {return fmt.Errorf("failed to inspect network %s: %w", rawNetwork.Name, inspectErr)}@@ -219,14 +215,10 @@ func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topol}func (s *NetworkService) CreateNetwork(ctx context.Context, name string, options client.NetworkCreateOptions, user usertypes.Actor) (*network.CreateResponse, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", "", name, user.ID, user.Username, "0", err, database.JSON{"action": "create", "driver": options.Driver})- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-defer s.eventService.BeginDockerResourceSuppressionWindow("network", "", name)()- response, err := dockerClient.NetworkCreate(ctx, name, options)+ response, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkCreateResult, error) {+ return dockerClient.NetworkCreate(ctx, name, options)+ })if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", "", name, user.ID, user.Username, "0", err, database.JSON{"action": "create", "driver": options.Driver})return nil, fmt.Errorf("failed to create network: %w", err)@@ -248,17 +240,15 @@ func (s *NetworkService) CreateNetwork(ctx context.Context, name string, options}func (s *NetworkService) RemoveNetwork(ctx context.Context, id string, user usertypes.Actor) error {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", id, "", user.ID, user.Username, "0", err, database.JSON{"action": "delete"})- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- networkInfo, err := compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})+ networkInfo, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {+ return compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})+ })networkName := kit.Ternary(err == nil, networkInfo.Network.Name, id)defer s.eventService.BeginDockerResourceSuppressionWindow("network", id, networkName)()- if _, networkRemoveErr := dockerClient.NetworkRemove(ctx, id, client.NetworkRemoveOptions{}); networkRemoveErr != nil {+ if _, networkRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkRemoveResult, error) {+ return dockerClient.NetworkRemove(ctx, id, client.NetworkRemoveOptions{})+ }); networkRemoveErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", id, networkName, user.ID, user.Username, "0", networkRemoveErr, database.JSON{"action": "delete"})return fmt.Errorf("failed to remove network: %w", networkRemoveErr)}@@ -276,12 +266,6 @@ func (s *NetworkService) RemoveNetwork(ctx context.Context, id string, user user// ConnectContainer connects a container to a network without recreating it.func (s *NetworkService) ConnectContainer(ctx context.Context, networkID string, req networktypes.ConnectContainerRequest, user usertypes.Actor) error {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", networkID, "", user.ID, user.Username, "0", err, database.JSON{"action": "connect", "containerId": req.ContainerID})- return fmt.Errorf("failed to connect to Docker: %w", err)- }-ipv4, ipv4Err := netip.ParseAddr(strings.TrimSpace(req.IPv4Address))if ipv4Err != nil && strings.TrimSpace(req.IPv4Address) != "" {return common.Classify(common.ErrValidation, fmt.Errorf("invalid IPv4 address %s: %w", strings.TrimSpace(req.IPv4Address), ipv4Err))@@ -291,20 +275,22 @@ func (s *NetworkService) ConnectContainer(ctx context.Context, networkID string,return common.Classify(common.ErrValidation, fmt.Errorf("invalid IPv6 address %s: %w", strings.TrimSpace(req.IPv6Address), ipv6Err))}- if _, networkConnectErr := dockerClient.NetworkConnect(ctx, networkID, client.NetworkConnectOptions{- Container: req.ContainerID,- EndpointConfig: &network.EndpointSettings{- Aliases: req.Aliases,- IPAMConfig: func() *network.EndpointIPAMConfig {- if !ipv4.IsValid() && !ipv6.IsValid() {- return nil- }- return &network.EndpointIPAMConfig{- IPv4Address: ipv4,- IPv6Address: ipv6,- }- }(),- },+ if _, networkConnectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkConnectResult, error) {+ return dockerClient.NetworkConnect(ctx, networkID, client.NetworkConnectOptions{+ Container: req.ContainerID,+ EndpointConfig: &network.EndpointSettings{+ Aliases: req.Aliases,+ IPAMConfig: func() *network.EndpointIPAMConfig {+ if !ipv4.IsValid() && !ipv6.IsValid() {+ return nil+ }+ return &network.EndpointIPAMConfig{+ IPv4Address: ipv4,+ IPv6Address: ipv6,+ }+ }(),+ },+ })}); networkConnectErr != nil {s.eventService.LogErrorEvent(ctx,@@ -338,15 +324,11 @@ func (s *NetworkService) ConnectContainer(ctx context.Context, networkID string,// DisconnectContainer disconnects a container from a network.func (s *NetworkService) DisconnectContainer(ctx context.Context, networkID string, req networktypes.DisconnectContainerRequest, user usertypes.Actor) error {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", networkID, "", user.ID, user.Username, "0", err, database.JSON{"action": "disconnect", "containerId": req.ContainerID})- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, networkDisconnectErr := dockerClient.NetworkDisconnect(ctx, networkID, client.NetworkDisconnectOptions{- Container: req.ContainerID,- Force: req.Force,+ if _, networkDisconnectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkDisconnectResult, error) {+ return dockerClient.NetworkDisconnect(ctx, networkID, client.NetworkDisconnectOptions{+ Container: req.ContainerID,+ Force: req.Force,+ })}); networkDisconnectErr != nil {s.eventService.LogErrorEvent(ctx,@@ -380,12 +362,9 @@ func (s *NetworkService) DisconnectContainer(ctx context.Context, networkID stri}func (s *NetworkService) PruneNetworks(ctx context.Context) (*network.PruneReport, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- report, err := dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: make(client.Filters)})+ report, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkPruneResult, error) {+ return dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: make(client.Filters)})+ })if err != nil {return nil, fmt.Errorf("failed to prune networks: %w", err)}@@ -402,11 +381,6 @@ func (s *NetworkService) PruneNetworks(ctx context.Context) (*network.PruneRepor}func (s *NetworkService) ListNetworksPaginated(ctx context.Context, params pagination.QueryParams) ([]networktypes.Summary, pagination.Response, networktypes.UsageCounts, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, pagination.Response{}, networktypes.UsageCounts{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-containers, err := s.dockerService.ListContainers(ctx)if err != nil {return nil, pagination.Response{}, networktypes.UsageCounts{}, fmt.Errorf("failed to list containers: %w", err)@@ -414,7 +388,9 @@ func (s *NetworkService) ListNetworksPaginated(ctx context.Context, params pagininUseByID, inUseByName := docker.BuildNetworkUsageMaps(containers)- networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ networkList, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkListResult, error) {+ return compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})+ })if err != nil {return nil, pagination.Response{}, networktypes.UsageCounts{}, fmt.Errorf("failed to list Docker networks: %w", err)}diff --git a/backend/internal/project/children/deployment/service.go b/backend/internal/project/children/deployment/service.goindex 8b877ad83..fd1848868 100644--- a/backend/internal/project/children/deployment/service.go+++ b/backend/internal/project/children/deployment/service.go@@ -24,8 +24,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/image"- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/projects""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")@@ -38,47 +36,19 @@ type Builder interface {// Service runs the image side of deployments: pulls, update reconciliation, builds and dependents.type Service struct {- settingsService *settings.SettingsService- imageService *image.ImageService- dockerService *docker.DockerClientService- buildService Builder+ imageService *image.ImageService+ dockerService *docker.DockerClientService+ buildService Builder}-func New(settingsService *settings.SettingsService, imageService *image.ImageService, dockerService *docker.DockerClientService, buildService Builder) *Service {+func New(imageService *image.ImageService, dockerService *docker.DockerClientService, buildService Builder) *Service {return &Service{- settingsService: settingsService,- imageService: imageService,- dockerService: dockerService,- buildService: buildService,+ imageService: imageService,+ dockerService: dockerService,+ buildService: buildService,}}-// Pull pulls one image within the configured pull timeout and reconciles its update state.-func (s *Service) Pull(- ctx context.Context,- imageRef string,- progressWriter io.Writer,- user usertypes.Actor,- credentials []containerregistry.Credential,-) error {- cfg := s.settingsService.GetSettingsConfig()-- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(cfg.DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))- defer pullCancel()-- if err := s.imageService.PullImage(pullCtx, imageRef, progressWriter, user, credentials); err != nil {- if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {- return fmt.Errorf("image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", imageRef)- }- return fmt.Errorf("failed to pull image %s: %w", imageRef, err)- }-- if err := s.imageService.ReconcilePulledImageUpdate(ctx, imageRef); err != nil {- slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", imageRef, "error", err)- }- return nil-}-// NamespaceDependents splits dependents into running and stopped-only services; never-created ones are skipped.// A ComposePs failure treats every dependent as running, since a dangling namespace reference is worse.func (s *Service) NamespaceDependents(ctx context.Context, compProj *composetypes.Project, services []string) (running, stopped []string) {@@ -114,8 +84,13 @@ func (s *Service) ImageOperations(user *usertypes.Actor, credentials []containerreturn projecttypes.ComposeImageOperations{Exists: s.imageService.ImageExistsLocally,LastTagged: s.imageService.ImageLastTagTime,- Pull: func(ctx context.Context, image string, progress io.Writer) error {- return s.Pull(ctx, image, progress, *cmp.Or(user, &usertypes.SystemUser), credentials)+ Pull: func(ctx context.Context, ref string, progress io.Writer) error {+ return s.imageService.Pull(ctx, image.PullRequest{+ Ref: ref,+ Progress: progress,+ User: *cmp.Or(user, &usertypes.SystemUser),+ Credentials: credentials,+ })},Build: func(ctx context.Context, request buildtypes.BuildRequest, progress io.Writer, service string) error {_, err := s.buildService.BuildImage(ctx, types.LocalDockerEnvironmentID, request, progress, service, user)diff --git a/backend/internal/project/children/lifecycle/service.go b/backend/internal/project/children/lifecycle/service.goindex e97a5d2d1..a84ef4844 100644--- a/backend/internal/project/children/lifecycle/service.go+++ b/backend/internal/project/children/lifecycle/service.go@@ -31,17 +31,14 @@ import (dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/image"- "github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/projects")// Service runs pre-deploy scripts in throwaway runner containers.type Service struct {- settingsService *settings.SettingsService- dockerService *dockerInternal.DockerClientService- imageService *image.ImageService+ dockerService *dockerInternal.DockerClientService+ imageService *image.ImageService}const (@@ -61,11 +58,10 @@ const (// lifecycleEnvKeyRegex enforces POSIX-style env keys in configured Env and script stdout.var lifecycleEnvKeyRegex = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)-func New(settingsService *settings.SettingsService, dockerService *dockerInternal.DockerClientService, imageService *image.ImageService) *Service {+func New(dockerService *dockerInternal.DockerClientService, imageService *image.ImageService) *Service {return &Service{- settingsService: settingsService,- dockerService: dockerService,- imageService: imageService,+ dockerService: dockerService,+ imageService: imageService,}}@@ -152,28 +148,22 @@ func (s *Service) RunScript(timeout time.Duration,actor user.Actor,) (stdoutContent, stderrContent string, exitCode int64, err error) {- dockerClient, dErr := s.dockerService.GetClient(ctx)- if dErr != nil {- return "", "", 0, fmt.Errorf("failed to connect to Docker: %w", dErr)- }-// Pull a missing runner image through the image service so credentials and the pull timeout apply.- if _, inspectErr := dockerClient.ImageInspect(ctx, runnerImage); inspectErr != nil {- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))- pullErr := s.imageService.PullImage(pullCtx, runnerImage, io.Discard, actor, nil)- pullTimedOut := errors.Is(pullCtx.Err(), context.DeadlineExceeded)- pullCancel()- if pullErr != nil && pullTimedOut {- return "", "", 0, fmt.Errorf("failed to ensure runner image %s: runner image pull timed out for %s (increase dockerImagePullTimeout setting if needed)", runnerImage, runnerImage)- }- if pullErr != nil {- return "", "", 0, fmt.Errorf("failed to ensure runner image %s: pull runner image %s: %w", runnerImage, runnerImage, pullErr)+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, runnerImage)+ }); inspectErr != nil {+ if pullErr := s.imageService.Pull(ctx, image.PullRequest{+ Ref: runnerImage,+ Progress: io.Discard,+ User: actor,+ }); pullErr != nil {+ return "", "", 0, fmt.Errorf("failed to ensure runner image %s: %w", runnerImage, pullErr)}}// MountForSubpath mounts a volume-backed /app/data via VolumeOptions.Subpath (host binds fail on e.g. WSL2).// On the host, or when the mount cannot be derived, projectPath is bound as-is.- inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ inspect, inspectErr := s.dockerService.Call(ctx, libarcane.InspectCurrentArcaneContainer)if inspectErr != nil {slog.WarnContext(ctx, "failed to derive workspace mount; falling back to bind on project path", "projectPath", projectPath, "error", inspectErr)}@@ -195,51 +185,54 @@ func (s *Service) RunScript(env = append(env, k+"="+v)}- createCtx, createCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer createCancel()- resp, err := dockerClient.ContainerCreate(createCtx, client.ContainerCreateOptions{- // Cmd is the script alone so its shebang picks the interpreter; Entrypoint is cleared for images like getsops/sops.- Config: &container.Config{- Image: runnerImage,- Entrypoint: []string{},- Cmd: []string{filepath.ToSlash(filepath.Join(lifecycleWorkspaceMount, scriptPath))},- WorkingDir: lifecycleWorkspaceMount,- Env: env,- AttachStdout: true,- AttachStderr: true,- Labels: map[string]string{- libarcane.InternalResourceLabel: "true",+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ // Cmd is the script alone so its shebang picks the interpreter; Entrypoint is cleared for images like getsops/sops.+ Config: &container.Config{+ Image: runnerImage,+ Entrypoint: []string{},+ Cmd: []string{filepath.ToSlash(filepath.Join(lifecycleWorkspaceMount, scriptPath))},+ WorkingDir: lifecycleWorkspaceMount,+ Env: env,+ AttachStdout: true,+ AttachStderr: true,+ Labels: map[string]string{+ libarcane.InternalResourceLabel: "true",+ },+ },+ // no-new-privileges and CapDrop ALL strip setuid and root capabilities from the script and its children.+ HostConfig: &container.HostConfig{+ Mounts: mounts,+ NetworkMode: container.NetworkMode(cmp.Or(strings.TrimSpace(networkMode), "none")),+ SecurityOpt: []string{"no-new-privileges:true"},+ CapDrop: []string{"ALL"},},- },- // no-new-privileges and CapDrop ALL strip setuid and root capabilities from the script and its children.- HostConfig: &container.HostConfig{- Mounts: mounts,- NetworkMode: container.NetworkMode(cmp.Or(strings.TrimSpace(networkMode), "none")),- SecurityOpt: []string{"no-new-privileges:true"},- CapDrop: []string{"ALL"},- },+ })})if err != nil {return "", "", 0, fmt.Errorf("create lifecycle container: %w", err)}defer func() {// Cleanup outlives cancellation so a cancelled run never leaks its container.- cleanupCtx, cleanupCancel := context.WithTimeout(- context.WithoutCancel(ctx),- timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI),- )- defer cleanupCancel()- if _, removeErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, client.ContainerRemoveOptions{Force: true}); removeErr != nil && !errdefs.IsNotFound(removeErr) {+ cleanupCtx := context.WithoutCancel(ctx)+ if _, removeErr := s.dockerService.Call(cleanupCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ }); removeErr != nil && !errdefs.IsNotFound(removeErr) {slog.WarnContext(cleanupCtx, "failed to remove lifecycle container", "containerId", resp.ID, "error", removeErr)}}()- startCtx, startCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer startCancel()- if _, containerStartErr := dockerClient.ContainerStart(startCtx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {return "", "", 0, fmt.Errorf("start lifecycle container: %w", containerStartErr)}+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return "", "", 0, fmt.Errorf("failed to connect to Docker: %w", err)+ }+logsCtx, logsCancel := context.WithCancel(ctx)defer logsCancel()logs, err := dockerClient.ContainerLogs(logsCtx, resp.ID, client.ContainerLogsOptions{diff --git a/backend/internal/project/service.go b/backend/internal/project/service.goindex f3eaf6fd8..cbb817b09 100644--- a/backend/internal/project/service.go+++ b/backend/internal/project/service.go@@ -20,6 +20,7 @@ import ("time""github.com/compose-spec/compose-go/v2/types"+ "github.com/containerd/errdefs""github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/containerregistry"imagetypes "github.com/getarcaneapp/arcane/types/v2/image"@@ -83,6 +84,8 @@ const (// inferredServiceCountBatchSize keeps one inferred count update under// SQLite's bound variable limit (three placeholders per id).inferredServiceCountBatchSize = 200+ // leaseWaitTimeout bounds how long a project operation waits for one already running on the same project.+ leaseWaitTimeout = 30 * time.Second)var (@@ -110,6 +113,8 @@ type ProjectService struct {// syncMu serializes SyncProjectsFromFileSystem: its discovery walk and its// cleanup pass must not interleave with another run's.syncMu sync.Mutex+ // opLock serializes mutating operations on the same project.+ opLock utils.KeyedMutexcomposeNames composeNameCacheparsedCompose projecttypes.ComposeCache[*types.Project]@@ -358,7 +363,7 @@ func NewProjectService(return buildUpdateInfoSummary(imageRefs, projectdetails.MergeProjectContainerUpdateInfo(nil, services, scoped), nil)},),- deployment: deployment.New(settingsService, imageService, dockerService, buildService),+ deployment: deployment.New(imageService, dockerService, buildService),// Rename recovery compares the stored name and path against its journal and rolls the row back.updates: update.New(kvService,@@ -396,6 +401,50 @@ func (s *ProjectService) RecoverProjectRenameJournals(ctx context.Context) errorreturn s.updates.RecoverAll(ctx)}+// RecoverStuckProjectStatuses re-derives the status of projects left Deploying or Restarting by a restart.+func (s *ProjectService) RecoverStuckProjectStatuses(ctx context.Context) error {+ var stuck []Project+ if err := s.db.WithContext(ctx).Where("status IN ?", []ProjectStatus{ProjectStatusDeploying, ProjectStatusRestarting}).Find(&stuck).Error; err != nil {+ return fmt.Errorf("list projects with interrupted status: %w", err)+ }++ var firstErr error+ recovered := 0+ for _, proj := range stuck {+ err := func() error {+ unlock, err := s.lockProject(ctx, "recover project status", proj.ID)+ if err != nil {+ return err+ }+ defer unlock()++ services, err := s.projectServices(ctx, proj.ID)+ if err != nil {+ return fmt.Errorf("load services for project %s: %w", proj.ID, err)+ }+ status := ProjectStatus(listing.ProjectStatus(services))+ if status == ProjectStatusUnknown {+ slog.WarnContext(ctx, "Leaving interrupted project status unchanged: runtime status unknown", "projectId", proj.ID, "status", proj.Status)+ return nil+ }+ if updateErr := s.updateProjectStatusAndCounts(ctx, proj.ID, status); updateErr != nil {+ return updateErr+ }+ recovered+++ return nil+ }()+ if err != nil {+ slog.WarnContext(ctx, "Failed to recover interrupted project status", "projectId", proj.ID, "error", err)+ firstErr = cmp.Or(firstErr, err)+ }+ }++ if len(stuck) > 0 {+ slog.InfoContext(ctx, "Recovered interrupted project statuses", "found", len(stuck), "recovered", recovered)+ }+ return firstErr+}+func (s *ProjectService) ResolveRegistryCredentials(ctx context.Context) ([]containerregistry.Credential, error) {credentials, err := s.RegistryCredentialsProvider(ctx)if err != nil {@@ -425,6 +474,17 @@ func (s *ProjectService) getMutableProject(ctx context.Context, projectID stringreturn proj, nil}+// lockProject takes the project's operation lease, waiting up to leaseWaitTimeout for an operation already running on it.+func (s *ProjectService) lockProject(ctx context.Context, op, projectID string) (func(), error) {+ waitCtx, cancel := context.WithTimeout(ctx, leaseWaitTimeout)+ defer cancel()+ unlock, err := s.opLock.LockContext(waitCtx, "0:project:"+projectID)+ if err != nil {+ return nil, fmt.Errorf("%s %s: waiting for in-progress operation: %w", op, projectID, err)+ }+ return unlock, nil+}+func (s *ProjectService) logProjectEvent(ctx context.Context, eventType event.EventType, projectID, projectName string, user usertypes.Actor, metadata database.JSON, action string) {if logErr := s.eventService.LogProjectEvent(ctx, eventType, projectID, projectName, user.ID, user.Username, "0", metadata); logErr != nil {slog.ErrorContext(ctx, "could not log project event", "action", action, "error", logErr)@@ -797,7 +857,7 @@ func NewLifecycleService(db: db,settingsService: settingsService,eventService: eventService,- hooks: lifecycle.New(settingsService, dockerService, imageService),+ hooks: lifecycle.New(dockerService, imageService),}}@@ -983,6 +1043,12 @@ func ParseExtraMountsText(raw *string) ([]lifecycletype.ExtraMount, error) {}func (s *ProjectService) UpdateProjectServices(ctx context.Context, projectID string, servicesToUpdate []string, user usertypes.Actor, discoverTags bool) error {+ unlock, err := s.lockProject(ctx, "update project services", projectID)+ if err != nil {+ return err+ }+ defer unlock()+proj, err := s.getMutableProject(ctx, projectID)if err != nil {return err@@ -1048,25 +1114,7 @@ func (s *ProjectService) UpdateProjectServices(ctx context.Context, projectID st}},Recover: func(ctx context.Context) {- services, servicesErr := s.projectServices(ctx, projectID)- if servicesErr != nil {- slog.WarnContext(ctx, "failed to inspect project services after deploy failure", "projectId", projectID, "error", servicesErr)- } else {- serviceCount, runningCount := listing.ServiceCounts(services)- updateErr := s.db.WithContext(ctx).Model(&Project{}).Where("id = ?", projectID).Updates(map[string]any{- "status": ProjectStatus(listing.ProjectStatus(services)),- "service_count": serviceCount,- "running_count": runningCount,- "updated_at": time.Now(),- }).Error- if updateErr == nil {- return- }- slog.WarnContext(ctx, "failed to restore project status after deploy failure", "projectId", projectID, "error", updateErr)- }- if updateErr := s.updateProjectStatus(ctx, projectID, ProjectStatusStopped); updateErr != nil {- slog.WarnContext(ctx, "failed to set stopped status after deploy failure", "projectId", projectID, "error", updateErr)- }+ s.restoreRuntimeProjectStatus(ctx, projectID)},}); updateServicesErr != nil {return updateServicesErr@@ -1159,6 +1207,15 @@ func (s *ProjectService) UnarchiveProject(ctx context.Context, projectID string,}func (s *ProjectService) DeployProject(ctx context.Context, projectID string, user usertypes.Actor, options *projecttypes.DeployOptions) error {+ unlock, err := s.lockProject(ctx, "deploy project", projectID)+ if err != nil {+ return err+ }+ defer unlock()+ return s.deployProject(ctx, projectID, user, options)+}++func (s *ProjectService) deployProject(ctx context.Context, projectID string, user usertypes.Actor, options *projecttypes.DeployOptions) error {projectFromDb, err := s.getMutableProject(ctx, projectID)switch {case errors.Is(err, common.ErrProjectArchived):@@ -1215,25 +1272,7 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us}, resolveRegistryCredentialsErr},Recover: func(ctx context.Context) {- services, servicesErr := s.projectServices(ctx, projectID)- if servicesErr != nil {- slog.WarnContext(ctx, "failed to inspect project services after deploy failure", "projectId", projectID, "error", servicesErr)- } else {- serviceCount, runningCount := listing.ServiceCounts(services)- updateErr := s.db.WithContext(ctx).Model(&Project{}).Where("id = ?", projectID).Updates(map[string]any{- "status": ProjectStatus(listing.ProjectStatus(services)),- "service_count": serviceCount,- "running_count": runningCount,- "updated_at": time.Now(),- }).Error- if updateErr == nil {- return- }- slog.WarnContext(ctx, "failed to restore project status after deploy failure", "projectId", projectID, "error", updateErr)- }- if updateErr := s.updateProjectStatus(ctx, projectID, ProjectStatusStopped); updateErr != nil {- slog.WarnContext(ctx, "failed to set stopped status after deploy failure", "projectId", projectID, "error", updateErr)- }+ s.restoreRuntimeProjectStatus(ctx, projectID)},})if err != nil {@@ -1254,14 +1293,24 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us}func (s *ProjectService) DownProject(ctx context.Context, projectID string, user usertypes.Actor) error {+ unlock, err := s.lockProject(ctx, "down project", projectID)+ if err != nil {+ return err+ }+ defer unlock()+projectFromDb, err := s.getMutableProject(ctx, projectID)if err != nil {return err}+ return s.downProject(ctx, projectFromDb, user)+}+// downProject brings the project down and restores its previous status on failure.+func (s *ProjectService) downProject(ctx context.Context, projectFromDb *Project, user usertypes.Actor) error {+ projectID := projectFromDb.IDproj, _, lerr := s.loadComposeProjectForProject(ctx, projectFromDb, nil)if lerr != nil {- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)return fmt.Errorf("failed to load compose project: %w", lerr)}@@ -1272,7 +1321,7 @@ func (s *ProjectService) DownProject(ctx context.Context, projectID string, userdefer s.eventService.BeginComposeSuppressionWindow(proj.Name)()if composeDownErr := projects.ComposeDown(ctx, proj, false); composeDownErr != nil {- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)+ _ = s.updateProjectStatus(ctx, projectID, projectFromDb.Status)return fmt.Errorf("failed to bring down project: %w", composeDownErr)}@@ -1478,6 +1527,12 @@ func (s *ProjectService) DestroyProject(ctx context.Context, projectID string, r"userId", user.ID,"username", user.Username)+ unlock, err := s.lockProject(ctx, "destroy project", projectID)+ if err != nil {+ return err+ }+ defer unlock()+proj, err := s.GetProjectFromDatabaseByID(ctx, projectID)if err != nil {return err@@ -1487,18 +1542,25 @@ func (s *ProjectService) DestroyProject(ctx context.Context, projectID string, r"projectName", proj.Name,"projectPath", proj.Path)- if downProjectErr := s.DownProject(ctx, projectID, usertypes.SystemUser); downProjectErr != nil {- slog.WarnContext(ctx, "failed to bring down project", "error", downProjectErr)+ // Teardown runs down, volumes, files, then the DB row, stopping at the first real failure.+ // Missing Docker resources and a missing compose file leave nothing to tear down.+ missing := func(err error) bool {+ return errdefs.IsNotFound(err) || errors.Is(err, common.ErrProjectComposeFileNotFound)+ }+ if downErr := s.downProject(ctx, proj, usertypes.SystemUser); downErr != nil && !missing(downErr) {+ return fmt.Errorf("failed to bring down project: %w", downErr)}if removeVolumes {- if compProj, _, lerr := s.loadComposeProjectForProject(ctx, proj, nil); lerr == nil {+ compProj, _, lerr := s.loadComposeProjectForProject(ctx, proj, nil)+ switch {+ case lerr == nil:defer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()- if derr := projects.ComposeDown(ctx, compProj, true); derr != nil {- slog.WarnContext(ctx, "failed to remove volumes", "error", derr)+ if derr := projects.ComposeDown(ctx, compProj, true); derr != nil && !missing(derr) {+ return fmt.Errorf("failed to remove project volumes: %w", derr)}- } else {- slog.WarnContext(ctx, "failed to load compose project for volume removal", "error", lerr)+ case !missing(lerr):+ return fmt.Errorf("failed to load compose project for volume removal: %w", lerr)}}@@ -1546,6 +1608,12 @@ func (s *ProjectService) DestroyProject(ctx context.Context, projectID string, r}func (s *ProjectService) RedeployProject(ctx context.Context, projectID string, user usertypes.Actor, options *projecttypes.DeployOptions) error {+ unlock, err := s.lockProject(ctx, "redeploy project", projectID)+ if err != nil {+ return err+ }+ defer unlock()+proj, err := s.getMutableProject(ctx, projectID)if err != nil {return err@@ -1578,7 +1646,7 @@ func (s *ProjectService) RedeployProject(ctx context.Context, projectID string,slog.WarnContext(ctx, "failed to pull project images", "error", pullProjectImagesErr)}- return s.DeployProject(ctx, projectID, user, options)+ return s.deployProject(ctx, projectID, user, options)}func (s *ProjectService) PullProjectImages(ctx context.Context, projectID string, progressWriter io.Writer, user usertypes.Actor, credentials []containerregistry.Credential) error {@@ -1595,7 +1663,12 @@ func (s *ProjectService) PullProjectImages(ctx context.Context, projectID stringdefer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()for _, imageRef := range projects.PullableImageRefs(compProj) {- if pullErr := s.deployment.Pull(ctx, imageRef, progressWriter, user, credentials); pullErr != nil {+ if pullErr := s.imageService.Pull(ctx, image.PullRequest{+ Ref: imageRef,+ Progress: progressWriter,+ User: user,+ Credentials: credentials,+ }); pullErr != nil {return pullErr}}@@ -1619,6 +1692,12 @@ func (s *ProjectService) BuildProjectServices(ctx context.Context, projectID str}func (s *ProjectService) RestartProject(ctx context.Context, projectID string, services []string, user usertypes.Actor) error {+ unlock, err := s.lockProject(ctx, "restart project", projectID)+ if err != nil {+ return err+ }+ defer unlock()+proj, err := s.getMutableProject(ctx, projectID)if err != nil {return err@@ -1627,20 +1706,19 @@ func (s *ProjectService) RestartProject(ctx context.Context, projectID string, sreturn resolveProjectComposeFileErr}- if updateProjectStatusErr := s.updateProjectStatus(ctx, projectID, ProjectStatusRestarting); updateProjectStatusErr != nil {- return fmt.Errorf("failed to update project status to restarting: %w", updateProjectStatusErr)- }-compProj, _, lerr := s.loadComposeProjectForProject(ctx, proj, nil)if lerr != nil {- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)return fmt.Errorf("failed to load compose project: %w", lerr)}+ if updateProjectStatusErr := s.updateProjectStatus(ctx, projectID, ProjectStatusRestarting); updateProjectStatusErr != nil {+ return fmt.Errorf("failed to update project status to restarting: %w", updateProjectStatusErr)+ }+defer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()if composeRestartErr := projects.ComposeRestart(ctx, compProj, services); composeRestartErr != nil {- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)+ _ = s.updateProjectStatus(ctx, projectID, proj.Status)return fmt.Errorf("failed to restart project: %w", composeRestartErr)}@@ -1676,6 +1754,20 @@ func (s *ProjectService) updateProjectStatusAndCounts(ctx context.Context, projereturn nil}+// restoreRuntimeProjectStatus re-derives status from the running services after a failed deploy, falling back to Stopped.+func (s *ProjectService) restoreRuntimeProjectStatus(ctx context.Context, projectID string) {+ services, err := s.projectServices(ctx, projectID)+ if err == nil {+ if err = s.updateProjectStatusAndCounts(ctx, projectID, ProjectStatus(listing.ProjectStatus(services))); err == nil {+ return+ }+ }+ slog.WarnContext(ctx, "failed to restore project status after deploy failure", "projectId", projectID, "error", err)+ if updateErr := s.updateProjectStatus(ctx, projectID, ProjectStatusStopped); updateErr != nil {+ slog.WarnContext(ctx, "failed to set stopped status after deploy failure", "projectId", projectID, "error", updateErr)+ }+}+func (s *ProjectService) updateProjectStatus(ctx context.Context, id string, status ProjectStatus) error {if err := s.db.WithContext(ctx).Model(&Project{}).Where("id = ?", id).Updates(map[string]any{"status": status,@@ -2734,6 +2826,12 @@ func (s *ProjectService) reconcileComposeTagsForProject(ctx context.Context, pro}func (s *ProjectService) UpdateProject(ctx context.Context, projectID string, name, composeContent, envContent, overrideContent *string, user usertypes.Actor) (*Project, error) {+ unlock, err := s.lockProject(ctx, "update project", projectID)+ if err != nil {+ return nil, err+ }+ defer unlock()+recoverErr := s.updates.RecoverProject(ctx, projectID)proj, err := s.getMutableProject(ctx, projectID)if err != nil {@@ -2945,6 +3043,12 @@ func (s *ProjectService) ApplyGitSyncProjectFiles(gitOverrideFileName string,user usertypes.Actor,) (*Project, bool, error) {+ unlock, err := s.lockProject(ctx, "apply git sync project files", projectID)+ if err != nil {+ return nil, false, err+ }+ defer unlock()+proj, err := s.getMutableProject(ctx, projectID)if err != nil {return nil, false, err@@ -3202,6 +3306,12 @@ func (s *ProjectService) UpdateProjectWorkspace(if err := workspacepkg.ValidateUpdateManifest(manifest.FileTreeRevision, len(manifest.FileChanges), 500); err != nil {return nil, common.Classify(common.ErrProjectWorkspaceBadRequest, err)}+ unlock, err := s.lockProject(ctx, "update project workspace", projectID)+ if err != nil {+ return nil, err+ }+ defer unlock()+proj, err := s.getMutableProject(ctx, projectID)if err != nil {return nil, errdiff --git a/backend/internal/project/service_test.go b/backend/internal/project/service_test.goindex 385c3fee8..619d0b656 100644--- a/backend/internal/project/service_test.go+++ b/backend/internal/project/service_test.go@@ -417,6 +417,93 @@ func TestProjectService_DestroyProject_PreservesFilesWhenRequested(t *testing.T)assert.FileExists(t, filepath.Join(projectsDir, trashName, "project-data.txt"))}+func TestProjectService_LockProject_SerializesSameProject(t *testing.T) {+ svc, _, _ := setupProjectDestroyTestService(t)++ unlock, err := svc.lockProject(t.Context(), "test", "project-a")+ require.NoError(t, err)++ done := make(chan error, 1)+ go func() { done <- svc.DownProject(t.Context(), "project-a", usertypes.Actor{}) }()++ select {+ case downErr := <-done:+ t.Fatalf("DownProject ran while another operation held the project lease: %v", downErr)+ case <-time.After(100 * time.Millisecond):+ }++ otherCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ otherErr := svc.DownProject(otherCtx, "project-b", usertypes.Actor{})+ require.Error(t, otherErr)+ require.NotErrorIs(t, otherErr, context.DeadlineExceeded, "a different project must not wait on the held lease")++ unlock()+ select {+ case downErr := <-done:+ require.Error(t, downErr)+ require.NotErrorIs(t, downErr, context.DeadlineExceeded)+ case <-time.After(2 * time.Second):+ t.Fatal("DownProject did not run after the project lease was released")+ }+}++func TestProjectService_LockProject_ReturnsWaitErrorOnDeadline(t *testing.T) {+ svc, _, _ := setupProjectDestroyTestService(t)++ unlock, err := svc.lockProject(t.Context(), "test", "project-a")+ require.NoError(t, err)+ defer unlock()++ ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)+ defer cancel()+ downErr := svc.DownProject(ctx, "project-a", usertypes.Actor{})+ require.ErrorIs(t, downErr, context.DeadlineExceeded)+ assert.Contains(t, downErr.Error(), "down project project-a: waiting for in-progress operation")+}++func TestProjectService_DownProject_RestoresPreviousStatusOnFailure(t *testing.T) {+ ctx := t.Context()+ svc, db, projectsDir := setupProjectDestroyTestService(t)+ t.Setenv("DOCKER_HOST", "tcp://127.0.0.1:1")++ require.NoError(t, db.Create(&Project{+ ID: "project-down-fails",+ Name: "down-fails",+ DirName: new("down-fails"),+ Path: createComposeProjectDir(t, projectsDir, "down-fails"),+ Status: ProjectStatusStopped,+ }).Error)++ require.Error(t, svc.DownProject(ctx, "project-down-fails", usertypes.Actor{}))++ var stored Project+ require.NoError(t, db.First(&stored, "id = ?", "project-down-fails").Error)+ assert.Equal(t, ProjectStatusStopped, stored.Status)+}++func TestProjectService_DestroyProject_KeepsRowWhenDownFails(t *testing.T) {+ ctx := t.Context()+ svc, db, projectsDir := setupProjectDestroyTestService(t)+ t.Setenv("DOCKER_HOST", "tcp://127.0.0.1:1")++ projectPath := createComposeProjectDir(t, projectsDir, "destroy-down-fails")+ require.NoError(t, db.Create(&Project{+ ID: "project-destroy-down-fails",+ Name: "destroy-down-fails",+ DirName: new("destroy-down-fails"),+ Path: projectPath,+ Status: ProjectStatusRunning,+ }).Error)++ require.Error(t, svc.DestroyProject(ctx, "project-destroy-down-fails", true, true, usertypes.Actor{}))++ var stored Project+ require.NoError(t, db.First(&stored, "id = ?", "project-destroy-down-fails").Error)+ assert.Equal(t, ProjectStatusRunning, stored.Status)+ assert.DirExists(t, projectPath)+}+func TestProjectService_GetProjectFromDatabaseByID(t *testing.T) {db := setupProjectTestDB(t)ctx := t.Context()@@ -701,7 +788,7 @@ func TestProjectService_PullProjectImages_UpdatesCurrentImageRecordAfterPull(t *dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}eventService := event.NewEventService(db, nil, nil)imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)svc := NewProjectService(db,settingsService,@@ -809,7 +896,7 @@ func TestProjectService_EnsureImagesPresent_UpdatesCurrentImageRecordAfterPull(tdockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}eventService := event.NewEventService(db, nil, nil)imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)svc := NewProjectService(db,settingsService,@@ -889,7 +976,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(tdockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}eventService := event.NewEventService(db, nil, nil)imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)svc := NewProjectService(db,settingsService,@@ -914,7 +1001,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(tCheckTime: time.Now().UTC().Add(-time.Hour),}).Error)- require.NoError(t, svc.deployment.Pull(ctx, imageRef, io.Discard, usertypes.SystemUser, nil))+ require.NoError(t, svc.deployment.ImageOperations(nil, nil).Pull(ctx, imageRef, io.Discard))// sha256:old-worker may still be in use by another container — must not be cleared (fixes #2453).var oldRecord imageupdate.ImageUpdateRecord@@ -964,7 +1051,7 @@ func TestProjectService_ComposePullSelectedServices_ReconcilesOnlyOnSuccess(t *tdockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}eventService := event.NewEventService(db, nil, nil)imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)svc := NewProjectService(db,settingsService,@@ -1080,7 +1167,7 @@ func TestProjectService_ComposePullSelectedServices_LeavesRecordsWhenPullFails(tdockerService := &docker.DockerClientService{Client: newTestDockerClient(t, failingServer)}imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), settingsService, nil)svc := NewProjectService(db,settingsService,@@ -1150,7 +1237,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFails(t *testing.TdockerService := &docker.DockerClientService{Client: newTestDockerClient(t, failingServer)}imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), settingsService, nil)projectPath := createComposeProjectDir(t, projectsDir, "compose-update-pull-fail")require.NoError(@@ -1250,7 +1337,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreate(t *testing.T) {dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)eventService := event.NewEventService(db, nil, nil)- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)projectPath := createComposeProjectDir(t, projectsDir, "compose-update-force")require.NoError(@@ -2155,6 +2242,85 @@ func TestProjectService_UpdateProject_RenameRejectsStaleStoppedWhenRuntimeIsRunnassert.Equal(t, ProjectStatusStopped, fromDB.Status)}+func TestProjectService_RecoverStuckProjectStatuses(t *testing.T) {+ tests := []struct {+ name string+ saved ProjectStatus+ containers []container.Summary+ want ProjectStatus+ }{+ {+ name: "deploying with running containers becomes running",+ saved: ProjectStatusDeploying,+ containers: []container.Summary{+ {+ ID: "app-container",+ Names: []string{"/foo-app-1"},+ Image: "nginx:alpine",+ State: container.StateRunning,+ Status: "Up 30 seconds",+ Labels: map[string]string{+ api.ProjectLabel: "foo",+ api.ServiceLabel: "app",+ },+ },+ },+ want: ProjectStatusRunning,+ },+ {+ // Compose services without a container are reported as stopped.+ name: "restarting without containers becomes stopped",+ saved: ProjectStatusRestarting,+ want: ProjectStatusStopped,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ db := setupProjectTestDB(t)+ ctx := t.Context()++ projectsDir := t.TempDir()+ t.Setenv("PROJECTS_DIRECTORY", projectsDir)++ settingsService, err := newSettingsServiceForTest(t, ctx, db)+ require.NoError(t, err)+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))++ server := configureProjectRuntimeDocker(t, tt.containers)+ svc := NewProjectService(+ db,+ settingsService,+ event.NewEventService(db, nil, nil),+ image.NewImageService(db, nil, nil, nil, nil, nil, nil, nil),+ &docker.DockerClientService{Client: newTestDockerClient(t, server)},+ testBuildBuilder{},+ nil,+ nil,+ config.Load(),+ kv.NewKVService(db),+ testRegistryCredentials,+ )++ dirName := "foo"+ project := &Project{+ ID: "proj-stuck",+ Name: "foo",+ DirName: &dirName,+ Path: createComposeProjectDir(t, projectsDir, dirName),+ Status: tt.saved,+ }+ require.NoError(t, db.Create(project).Error)++ require.NoError(t, svc.RecoverStuckProjectStatuses(ctx))++ var fromDB Project+ require.NoError(t, db.First(&fromDB, "id = ?", project.ID).Error)+ assert.Equal(t, tt.want, fromDB.Status)+ })+ }+}+func TestProjectService_UpdateProject_RenameResolvesUnknownStoppedStatusBeforeVolumeMigration(t *testing.T) {db := setupProjectTestDB(t)ctx := t.Context()@@ -8670,7 +8836,7 @@ func TestUpdateProjectServiceImagesPersistsBeforeDeploymentAndRetries(t *testingdb,settingsService,event.NewEventService(db, nil, nil),- image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), nil, nil),+ image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), settingsService, nil),testDockerService,testBuildBuilder{},nil,@@ -9058,7 +9224,7 @@ func TestProjectServiceManualUpdateDiscoversTags(t *testing.T) {db,settingsService,event.NewEventService(db, nil, nil),- image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), nil, nil),+ image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), settingsService, nil),testDockerService,testBuildBuilder{},nil,diff --git a/backend/internal/swarm/children/configs/service.go b/backend/internal/swarm/children/configs/service.goindex a6b5b02d0..af7dcdae8 100644--- a/backend/internal/swarm/children/configs/service.go+++ b/backend/internal/swarm/children/configs/service.go@@ -2,7 +2,6 @@ package configsimport ("context"- stdjson "encoding/json""encoding/json/v2""errors""fmt"@@ -11,16 +10,18 @@ import (swarmtypes "github.com/getarcaneapp/arcane/types/v2/swarm""github.com/moby/moby/api/types/swarm""github.com/moby/moby/client"++ "github.com/getarcaneapp/arcane/backend/v2/internal/docker")// Service manages swarm configs.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)+ docker *docker.DockerClientServiceensureManager func(ctx context.Context) error}-func NewService(dockerClient func(ctx context.Context) (*client.Client, error), ensureManager func(ctx context.Context) error) *Service {- return &Service{dockerClient: dockerClient, ensureManager: ensureManager}+func NewService(dockerService *docker.DockerClientService, ensureManager func(ctx context.Context) error) *Service {+ return &Service{docker: dockerService, ensureManager: ensureManager}}func (s *Service) ListConfigs(ctx context.Context) ([]swarmtypes.ConfigSummary, error) {@@ -28,12 +29,9 @@ func (s *Service) ListConfigs(ctx context.Context) ([]swarmtypes.ConfigSummary,return nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- configsResult, err := dockerClient.ConfigList(ctx, client.ConfigListOptions{})+ configsResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigListResult, error) {+ return dockerClient.ConfigList(ctx, client.ConfigListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list swarm configs: %w", err)}@@ -50,12 +48,9 @@ func (s *Service) GetConfig(ctx context.Context, configID string) (*swarmtypes.Creturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- cfgResult, err := dockerClient.ConfigInspect(ctx, configID, client.ConfigInspectOptions{})+ cfgResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigInspectResult, error) {+ return dockerClient.ConfigInspect(ctx, configID, client.ConfigInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm config: %w", err)}@@ -68,17 +63,20 @@ func (s *Service) CreateConfig(ctx context.Context, req swarmtypes.ConfigCreateRreturn nil, err}- spec, err := decodeConfigSpecInternal(req.Spec)- if err != nil {- return nil, err+ if trimmed := strings.TrimSpace(string(req.Spec)); trimmed == "" || trimmed == "null" {+ return nil, errors.New("config spec is required")}-- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)+ var spec swarm.ConfigSpec+ if err := json.Unmarshal(req.Spec, &spec); err != nil {+ return nil, fmt.Errorf("failed to parse config spec: %w", err)+ }+ if strings.TrimSpace(spec.Name) == "" {+ return nil, errors.New("config spec name is required")}- createResult, err := dockerClient.ConfigCreate(ctx, client.ConfigCreateOptions{Spec: spec})+ createResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigCreateResult, error) {+ return dockerClient.ConfigCreate(ctx, client.ConfigCreateOptions{Spec: spec})+ })if err != nil {return nil, fmt.Errorf("failed to create swarm config: %w", err)}@@ -91,31 +89,11 @@ func (s *Service) RemoveConfig(ctx context.Context, configID string) error {return err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, configRemoveErr := dockerClient.ConfigRemove(ctx, configID, client.ConfigRemoveOptions{}); configRemoveErr != nil {+ if _, configRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigRemoveResult, error) {+ return dockerClient.ConfigRemove(ctx, configID, client.ConfigRemoveOptions{})+ }); configRemoveErr != nil {return fmt.Errorf("failed to remove swarm config: %w", configRemoveErr)}return nil}--func decodeConfigSpecInternal(raw stdjson.RawMessage) (swarm.ConfigSpec, error) {- if len(raw) == 0 || strings.TrimSpace(string(raw)) == "" || strings.TrimSpace(string(raw)) == "null" {- return swarm.ConfigSpec{}, errors.New("config spec is required")- }-- var spec swarm.ConfigSpec- if err := json.Unmarshal(raw, &spec); err != nil {- return swarm.ConfigSpec{}, fmt.Errorf("failed to parse config spec: %w", err)- }-- if strings.TrimSpace(spec.Name) == "" {- return swarm.ConfigSpec{}, errors.New("config spec name is required")- }-- return spec, nil-}diff --git a/backend/internal/swarm/children/nodes/service.go b/backend/internal/swarm/children/nodes/service.goindex eb2dc3f51..f67c30215 100644--- a/backend/internal/swarm/children/nodes/service.go+++ b/backend/internal/swarm/children/nodes/service.go@@ -9,6 +9,7 @@ import ("fmt""log/slog""net/http"+ "slices""strings""sync""time"@@ -17,10 +18,12 @@ import ("github.com/moby/moby/api/types/swarm""github.com/moby/moby/client""github.com/samber/hot"+ "github.com/samber/mo""go.getarcane.app/kit/pkg""golang.org/x/sync/errgroup""github.com/getarcaneapp/arcane/backend/v2/internal/common"+ dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"@@ -35,7 +38,7 @@ const (// Service manages swarm nodes, their agent bindings, and Easy Join.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)+ docker *dockerInternal.DockerClientServiceensureManager func(ctx context.Context) errorlistTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error)localJoinTokens func(ctx context.Context) (*swarmtypes.SwarmJoinTokensResponse, error)@@ -44,14 +47,14 @@ type Service struct {}func NewService(- dockerClient func(ctx context.Context) (*client.Client, error),+ dockerService *dockerInternal.DockerClientService,ensureManager func(ctx context.Context) error,listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error),localJoinTokens func(ctx context.Context) (*swarmtypes.SwarmJoinTokensResponse, error),environmentService *environment.EnvironmentService,) *Service {return &Service{- dockerClient: dockerClient,+ docker: dockerService,ensureManager: ensureManager,listTasks: listTasks,localJoinTokens: localJoinTokens,@@ -84,7 +87,7 @@ type swarmNodeAgentCoverage struct {candidatesByNodeID map[string][]environment.EnvironmentlocalIdentity *SwarmNodeIdentity// Resolved once per request rather than per node: the name is user-editable, and- // applyNodeAgentCoverageInternal runs in a loop with no context to look it up.+ // applyNodeAgentCoverage runs in a loop with no context to look it up.localEnvironmentName string}@@ -101,8 +104,8 @@ func (s *Service) ListNodesPaginated(ctx context.Context, environmentID string,return nil, pagination.Response{}, errors.New("remote swarm node listing failed")}- s.enrichNodeAgentStatusesInternal(ctx, environmentID, remote.Data)- result := s.buildNodePaginationConfigInternal().SearchOrderAndPaginate(remote.Data, params)+ s.enrichNodeAgentStatuses(ctx, environmentID, remote.Data)+ result := s.buildNodePaginationConfig().SearchOrderAndPaginate(remote.Data, params)return result.Items, pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params), nil}@@ -110,12 +113,9 @@ func (s *Service) ListNodesPaginated(ctx context.Context, environmentID string,return nil, pagination.Response{}, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})+ nodesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {+ return dockerClient.NodeList(ctx, client.NodeListOptions{})+ })if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list swarm nodes: %w", err)}@@ -126,9 +126,9 @@ func (s *Service) ListNodesPaginated(ctx context.Context, environmentID string,items = append(items, swarmtypes.NewNodeSummary(node))}- s.enrichNodeAgentStatusesInternal(ctx, environmentID, items)+ s.enrichNodeAgentStatuses(ctx, environmentID, items)- config := s.buildNodePaginationConfigInternal()+ config := s.buildNodePaginationConfig()result := config.SearchOrderAndPaginate(items, params)paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params)@@ -149,7 +149,7 @@ func (s *Service) GetNode(ctx context.Context, environmentID, nodeID string) (*sreturn nil, errors.New("remote swarm node inspection failed")}items := []swarmtypes.NodeSummary{remote.Data}- s.enrichNodeAgentStatusesInternal(ctx, environmentID, items)+ s.enrichNodeAgentStatuses(ctx, environmentID, items)return &items[0], nil}@@ -157,18 +157,15 @@ func (s *Service) GetNode(ctx context.Context, environmentID, nodeID string) (*sreturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- nodeResult, err := dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})+ nodeResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeInspectResult, error) {+ return dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm node: %w", err)}items := []swarmtypes.NodeSummary{swarmtypes.NewNodeSummary(nodeResult.Node)}- s.enrichNodeAgentStatusesInternal(ctx, environmentID, items)+ s.enrichNodeAgentStatuses(ctx, environmentID, items)return new(items[0]), nil}@@ -201,10 +198,11 @@ func (s *Service) ReconcileNodeAgents(ctx context.Context, environmentID string)continue}- result.State = swarmtypes.NodeAgentStateConnected- result.EnvironmentID = &bound.ID- result.Candidates = nil- results = append(results, result)+ results = append(results, swarmtypes.NodeAgentReconcileResult{+ NodeID: node.ID,+ State: swarmtypes.NodeAgentStateConnected,+ EnvironmentID: &bound.ID,+ })}return &swarmtypes.NodeAgentReconcileResponse{Results: results}, nil@@ -221,7 +219,7 @@ func (s *Service) BindNodeAgent(ctx context.Context, parentEnvironmentID, nodeIDreturn nil, errors.New("only visible environments can be attached")}- runtime := s.resolveSwarmNodeAgentRuntimeInternal(ctx, localEnvironment)+ runtime := s.resolveSwarmNodeAgentRuntime(ctx, localEnvironment)if runtime.identity == nil || !runtime.identity.SwarmActive {return nil, errors.New("environment did not report an active swarm identity")}@@ -237,12 +235,9 @@ func (s *Service) BindNodeAgent(ctx context.Context, parentEnvironmentID, nodeID}func (s *Service) GetLocalNodeIdentity(ctx context.Context) (*SwarmNodeIdentity, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})+ infoResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SystemInfoResult, error) {+ return dockerClient.Info(ctx, client.InfoOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect local Docker engine: %w", err)}@@ -263,11 +258,11 @@ func (s *Service) GetLocalNodeIdentity(ctx context.Context) (*SwarmNodeIdentity,}, nil}-func (s *Service) enrichNodeAgentStatusesInternal(ctx context.Context, environmentID string, items []swarmtypes.NodeSummary) {+func (s *Service) enrichNodeAgentStatuses(ctx context.Context, environmentID string, items []swarmtypes.NodeSummary) {if s.environmentService == nil || len(items) == 0 || strings.TrimSpace(environmentID) == "" {return}- coverage, err := s.resolveNodeAgentCoverageInternal(ctx, environmentID)+ coverage, err := s.resolveNodeAgentCoverage(ctx, environmentID)if err != nil {slog.WarnContext(ctx, "failed to resolve swarm node agent coverage", "environmentId", environmentID, "error", err.Error())return@@ -275,11 +270,11 @@ func (s *Service) enrichNodeAgentStatusesInternal(ctx context.Context, environmefor index := range items {item := &items[index]- s.applyNodeAgentCoverageInternal(environmentID, item, coverage)+ s.applyNodeAgentCoverage(environmentID, item, coverage)}}-func (s *Service) resolveNodeAgentCoverageInternal(ctx context.Context, environmentID string) (*swarmNodeAgentCoverage, error) {+func (s *Service) resolveNodeAgentCoverage(ctx context.Context, environmentID string) (*swarmNodeAgentCoverage, error) {agentEnvs, err := s.environmentService.ListSwarmNodeAgentEnvironments(ctx, environmentID)if err != nil {return nil, err@@ -308,7 +303,7 @@ func (s *Service) resolveNodeAgentCoverageInternal(ctx context.Context, environmg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "swarm worker")- runtime := s.resolveSwarmNodeAgentRuntimeInternal(groupCtx, &env)+ runtime := s.resolveSwarmNodeAgentRuntime(groupCtx, &env)runtimeMu.Lock()runtimeByEnvID[env.ID] = runtimeruntimeMu.Unlock()@@ -354,99 +349,60 @@ func (s *Service) resolveNodeAgentCoverageInternal(ctx context.Context, environmreturn coverage, nil}-func (s *Service) applyNodeAgentCoverageInternal(environmentID string, item *swarmtypes.NodeSummary, coverage *swarmNodeAgentCoverage) {+func (s *Service) applyNodeAgentCoverage(environmentID string, item *swarmtypes.NodeSummary, coverage *swarmNodeAgentCoverage) {if item == nil || coverage == nil {return}nodeID := strings.TrimSpace(item.ID)if environmentID == environment.LocalEnvironmentID && coverage.localIdentity != nil && coverage.localIdentity.SwarmActive && strings.TrimSpace(coverage.localIdentity.SwarmNodeID) == nodeID {- connected := true- bindingKind := swarmtypes.NodeAgentBindingKindLocal- localID, localType := environment.LocalEnvironmentID, "local"- localName := environment.DisplayName(environment.LocalEnvironmentID, coverage.localEnvironmentName)item.Agent = swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateConnected,- BindingKind: &bindingKind,- EnvironmentID: &localID,- EnvironmentName: &localName,- EnvironmentType: &localType,- Connected: &connected,+ BindingKind: new(swarmtypes.NodeAgentBindingKindLocal),+ EnvironmentID: new(environment.LocalEnvironmentID),+ EnvironmentName: new(environment.DisplayName(environment.LocalEnvironmentID, coverage.localEnvironmentName)),+ EnvironmentType: new("local"),+ Connected: new(true),}return}boundEnvs := coverage.boundEnvsByNodeID[nodeID]if len(boundEnvs) > 0 {- localEnvironment := preferredNodeAgentEnvironmentInternal(boundEnvs)- item.Agent = s.buildNodeAgentStatusInternal(nodeID, &localEnvironment, coverage.runtimeByEnvID[localEnvironment.ID])+ // Prefer the first visible environment, falling back to the first one.+ localEnvironment := boundEnvs[max(0, slices.IndexFunc(boundEnvs, func(env environment.Environment) bool { return !env.Hidden }))]+ item.Agent = s.buildNodeAgentStatus(nodeID, &localEnvironment, coverage.runtimeByEnvID[localEnvironment.ID])return}candidates := coverage.candidatesByNodeID[nodeID]if len(candidates) > 1 {- item.Agent = swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateAmbiguous, Candidates: buildNodeAgentCandidatesInternal(candidates)}+ item.Agent = swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateAmbiguous, Candidates: buildNodeAgentCandidates(candidates)}return}if len(candidates) == 1 {- item.Agent.Candidates = buildNodeAgentCandidatesInternal(candidates)+ item.Agent.Candidates = buildNodeAgentCandidates(candidates)}}-func (s *Service) resolveSwarmNodeAgentRuntimeInternal(ctx context.Context, env *environment.Environment) swarmNodeAgentRuntime {+func (s *Service) resolveSwarmNodeAgentRuntime(ctx context.Context, env *environment.Environment) swarmNodeAgentRuntime {if env == nil {return swarmNodeAgentRuntime{}}- runtime := swarmNodeAgentRuntime{- connected: edge.HasActiveTunnel(env.ID),- }-- if tunnelState, ok := edge.GetTunnelRuntimeState(env.ID).Get(); ok {- runtime.lastHeartbeat = tunnelState.LastHeartbeat- }-- if pollState, ok := edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).Get(); ok {- runtime.lastPollAt = pollState.LastPollAt- }-- if identity := s.cachedSwarmNodeIdentityInternal(env.ID); identity != nil {- runtime.connected = true- runtime.identity = identity- return runtime- }-- identity, err := s.fetchSwarmNodeIdentityViaEdgeInternal(ctx, env.ID)- if err != nil {- slog.DebugContext(ctx, "failed to probe swarm node identity", "environmentId", env.ID, "error", err.Error())- return runtime- }-- runtime.connected = true- runtime.identity = identity- s.cacheSwarmNodeIdentityInternal(env.ID, *identity)- return runtime-}--func (s *Service) cachedSwarmNodeIdentityInternal(environmentID string) *SwarmNodeIdentity {- if s.identityCache == nil {- return nil- }- identity, ok, _ := s.identityCache.Get(environmentID)- if !ok {- return nil- }- return &identity-}--func (s *Service) cacheSwarmNodeIdentityInternal(environmentID string, identity SwarmNodeIdentity) {- if s.identityCache != nil {- s.identityCache.Set(environmentID, identity)+ identity, found, _ := s.identityCache.Get(env.ID)+ if !found {+ if fetched, err := s.fetchSwarmNodeIdentityViaEdge(ctx, env.ID); err != nil {+ slog.DebugContext(ctx, "failed to probe swarm node identity", "environmentId", env.ID, "error", err.Error())+ } else {+ identity, found = *fetched, true+ s.identityCache.Set(env.ID, identity)+ }}-}--func (s *Service) invalidateSwarmNodeIdentityInternal(environmentID string) {- if s.identityCache != nil {- s.identityCache.Delete(environmentID)+ return swarmNodeAgentRuntime{+ connected: found || edge.HasActiveTunnel(env.ID),+ lastHeartbeat: kit.FromPtr(edge.GetTunnelRuntimeState(env.ID).OrEmpty()).LastHeartbeat,+ lastPollAt: edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).OrEmpty().LastPollAt,+ identity: kit.Ternary(found, &identity, nil),}}@@ -461,11 +417,11 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID strif err != nil {return nil, err}- remoteAddrs, err := selectSwarmManagerAddressesInternal(request.RemoteAddrs, nodes)+ remoteAddrs, err := selectSwarmManagerAddresses(request.RemoteAddrs, nodes)if err != nil {return nil, err}- tokens, err := s.getSwarmJoinTokensForEnvironmentInternal(ctx, managerEnvironmentID)+ tokens, err := s.getSwarmJoinTokensForEnvironment(ctx, managerEnvironmentID)if err != nil {return nil, err}@@ -475,14 +431,14 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID str}results := make([]swarmtypes.SwarmJoinEnvironmentResult, len(request.Targets))- processTargetInternal := func(index int) {+ processTarget := func(index int) {target := request.Targets[index]- results[index] = s.joinEnvironmentInternal(ctx, managerEnvironmentID, target, remoteAddrs, tokens, memberNodeIDs)+ results[index] = s.joinEnvironment(ctx, managerEnvironmentID, target, remoteAddrs, tokens, memberNodeIDs)}for index, target := range request.Targets {if target.Role == swarmtypes.SwarmJoinEnvironmentRoleManager {- processTargetInternal(index)+ processTarget(index)}}@@ -496,7 +452,7 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID strdefer utils.RecoverToError(&workerErr, "swarm worker")if groupCtx.Err() == nil {- processTargetInternal(index)+ processTarget(index)}return nil})@@ -506,7 +462,7 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID strreturn &swarmtypes.SwarmJoinEnvironmentsResponse{Results: results}, nil}-func (s *Service) getSwarmJoinTokensForEnvironmentInternal(ctx context.Context, environmentID string) (*swarmtypes.SwarmJoinTokensResponse, error) {+func (s *Service) getSwarmJoinTokensForEnvironment(ctx context.Context, environmentID string) (*swarmtypes.SwarmJoinTokensResponse, error) {if environmentID == "0" {return s.localJoinTokens(ctx)}@@ -524,7 +480,7 @@ func (s *Service) getSwarmJoinTokensForEnvironmentInternal(ctx context.Context,return &response.Data, nil}-func (s *Service) joinEnvironmentInternal(+func (s *Service) joinEnvironment(ctx context.Context,managerEnvironmentID string,target swarmtypes.SwarmJoinEnvironmentTarget,@@ -532,97 +488,83 @@ func (s *Service) joinEnvironmentInternal(tokens *swarmtypes.SwarmJoinTokensResponse,memberNodeIDs map[string]struct{},) swarmtypes.SwarmJoinEnvironmentResult {- result := swarmtypes.SwarmJoinEnvironmentResult{EnvironmentID: target.EnvironmentID, State: swarmtypes.SwarmJoinEnvironmentResultFailed}- localEnvironment, err := s.environmentService.GetEnvironmentByID(ctx, target.EnvironmentID)- if err != nil || localEnvironment.Hidden || !localEnvironment.Enabled {- message := "target environment is unavailable"- if err != nil {- message = err.Error()+ result := func(state swarmtypes.SwarmJoinEnvironmentResultState, nodeID, message string) swarmtypes.SwarmJoinEnvironmentResult {+ return swarmtypes.SwarmJoinEnvironmentResult{+ EnvironmentID: target.EnvironmentID,+ State: state,+ NodeID: kit.Ternary(nodeID == "", nil, &nodeID),+ Error: kit.Ternary(message == "", nil, &message),}- result.Error = &message- return result+ }+ failed := func(message string) swarmtypes.SwarmJoinEnvironmentResult {+ return result(swarmtypes.SwarmJoinEnvironmentResultFailed, "", message)}- runtime := s.resolveSwarmNodeAgentRuntimeInternal(ctx, localEnvironment)+ localEnvironment, err := s.environmentService.GetEnvironmentByID(ctx, target.EnvironmentID)+ if err != nil {+ return failed(err.Error())+ }+ if localEnvironment.Hidden || !localEnvironment.Enabled {+ return failed("target environment is unavailable")+ }++ runtime := s.resolveSwarmNodeAgentRuntime(ctx, localEnvironment)if runtime.identity != nil && runtime.identity.SwarmActive {nodeID := strings.TrimSpace(runtime.identity.SwarmNodeID)if _, member := memberNodeIDs[nodeID]; !member {- message := "environment is active in another swarm cluster"- result.Error = &message- return result+ return failed("environment is active in another swarm cluster")}- if _, bindSwarmNodeEnvironmentErr := s.environmentService.BindSwarmNodeEnvironment(ctx, managerEnvironmentID, nodeID, target.EnvironmentID, true); bindSwarmNodeEnvironmentErr != nil {- message := bindSwarmNodeEnvironmentErr.Error()- result.Error = &message- return result+ if _, bindErr := s.environmentService.BindSwarmNodeEnvironment(ctx, managerEnvironmentID, nodeID, target.EnvironmentID, true); bindErr != nil {+ return failed(bindErr.Error())}- result.State = swarmtypes.SwarmJoinEnvironmentResultAlreadyMember- result.NodeID = &nodeID- return result+ return result(swarmtypes.SwarmJoinEnvironmentResultAlreadyMember, nodeID, "")}- joinToken := tokens.Worker- if target.Role == swarmtypes.SwarmJoinEnvironmentRoleManager {- joinToken = tokens.Manager- }- joinRequest := swarmtypes.SwarmJoinRequest{+ joinToken := kit.Ternary(target.Role == swarmtypes.SwarmJoinEnvironmentRoleManager, tokens.Manager, tokens.Worker)+ body, err := json.Marshal(swarmtypes.SwarmJoinRequest{ListenAddr: target.ListenAddr,AdvertiseAddr: target.AdvertiseAddr,DataPathAddr: target.DataPathAddr,RemoteAddrs: remoteAddrs,JoinToken: joinToken,Availability: target.Availability,- }- body, err := json.Marshal(joinRequest)+ })if err != nil {- message := err.Error()- result.Error = &message- return result+ return failed(err.Error())}var joinResponse struct {Success bool `json:"success"`}- if proxyJSONRequestErr := s.environmentService.ProxyJSONRequest(ctx, target.EnvironmentID, http.MethodPost, "/api/environments/0/swarm/join", body, &joinResponse); proxyJSONRequestErr != nil {- message := describeSwarmJoinFailureInternal(proxyJSONRequestErr, joinToken)- result.Error = &message- return result+ if proxyErr := s.environmentService.ProxyJSONRequest(ctx, target.EnvironmentID, http.MethodPost, "/api/environments/0/swarm/join", body, &joinResponse); proxyErr != nil {+ return failed(describeSwarmJoinFailure(proxyErr, joinToken))}if !joinResponse.Success {- message := "swarm join failed"- result.Error = &message- return result+ return failed("swarm join failed")}for range 6 {- s.invalidateSwarmNodeIdentityInternal(target.EnvironmentID)- verifiedRuntime := s.resolveSwarmNodeAgentRuntimeInternal(ctx, localEnvironment)+ s.identityCache.Delete(target.EnvironmentID)+ verifiedRuntime := s.resolveSwarmNodeAgentRuntime(ctx, localEnvironment)if verifiedRuntime.identity != nil && verifiedRuntime.identity.SwarmActive {nodeID := strings.TrimSpace(verifiedRuntime.identity.SwarmNodeID)if _, getNodeErr := s.GetNode(ctx, managerEnvironmentID, nodeID); getNodeErr == nil {if _, bindErr := s.environmentService.BindSwarmNodeEnvironment(ctx, managerEnvironmentID, nodeID, target.EnvironmentID, true); bindErr == nil {- result.State = swarmtypes.SwarmJoinEnvironmentResultJoined- result.NodeID = &nodeID- result.Error = nil- return result+ return result(swarmtypes.SwarmJoinEnvironmentResultJoined, nodeID, "")}}}select {case <-ctx.Done():- message := ctx.Err().Error()- result.Error = &message- return result+ return failed(ctx.Err().Error())case <-time.After(time.Second):}}- result.State = swarmtypes.SwarmJoinEnvironmentResultJoinedUnverified- result.Error = nil- return result+ return result(swarmtypes.SwarmJoinEnvironmentResultJoinedUnverified, "", "")}-func (s *Service) fetchSwarmNodeIdentityViaEdgeInternal(ctx context.Context, environmentID string) (*SwarmNodeIdentity, error) {+func (s *Service) fetchSwarmNodeIdentityViaEdge(ctx context.Context, environmentID string) (*SwarmNodeIdentity, error) {reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)defer cancel()@@ -640,69 +582,45 @@ func (s *Service) fetchSwarmNodeIdentityViaEdgeInternal(ctx context.Context, envreturn parsed.Data, nil}-func (s *Service) buildNodeAgentStatusInternal(nodeID string, env *environment.Environment, runtime swarmNodeAgentRuntime) swarmtypes.NodeAgentStatus {+func (s *Service) buildNodeAgentStatus(nodeID string, env *environment.Environment, runtime swarmNodeAgentRuntime) swarmtypes.NodeAgentStatus {if env == nil {return swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateNone}}- // An agent is live when it holds an open tunnel or has checked in via the poll- // control plane recently. lastHeartbeat/lastPollAt are only populated from the- // in-memory runtime registries when activity is fresh, so their presence is- // current evidence — unlike env.Status, which poll-mode agents never advance- // out of "pending" (HandlePoll updates the registry, not the persisted record).+ // Live means an open tunnel or a fresh poll check-in; the runtime registries only hold fresh activity,+ // unlike env.Status, which poll-mode agents never advance out of "pending".live := runtime.connected || runtime.lastHeartbeat != nil || runtime.lastPollAt != nil-- status := swarmtypes.NodeAgentStatus{- State: swarmtypes.NodeAgentStateOffline,- EnvironmentID: &env.ID,- Connected: &live,- LastHeartbeat: runtime.lastHeartbeat,- LastPollAt: runtime.lastPollAt,-- EnvironmentName: &env.Name,- }- environmentType := kit.Ternary(env.IsEdge, "edge", "direct")- status.EnvironmentType = &environmentType- bindingKind := kit.Ternary(env.Hidden, swarmtypes.NodeAgentBindingKindDedicated, swarmtypes.NodeAgentBindingKindEnvironment)- status.BindingKind = &bindingKind-- if runtime.identity != nil {- if reportedNodeID := strings.TrimSpace(runtime.identity.SwarmNodeID); reportedNodeID != "" {- status.ReportedNodeID = &reportedNodeID- }- if reportedHostname := strings.TrimSpace(runtime.identity.Hostname); reportedHostname != "" {- status.ReportedHostname = &reportedHostname- }- }-- // A live tunnel with a completed identity probe lets us authoritatively detect- // a node-ID mismatch (only tunnel mode can probe identity).- if runtime.connected && runtime.identity != nil {- if !runtime.identity.SwarmActive || strings.TrimSpace(runtime.identity.SwarmNodeID) != strings.TrimSpace(nodeID) {- status.State = swarmtypes.NodeAgentStateMismatched- return status- }- status.State = swarmtypes.NodeAgentStateConnected- return status- }-- // Poll-mode agents (and tunnels without a completed probe) can't be identity-- // verified live, but a fresh check-in proves the agent mapped to this node is- // reachable. The node-to-env mapping was established by Arcane at deploy time,- // so report it as connected instead of masking it behind a stale env.Status.- if live {- status.State = swarmtypes.NodeAgentStateConnected- return status- }-- // No live evidence: an unpaired agent stays pending; one seen before is offline.- if env.Status == string(environment.EnvironmentStatusPending) || env.LastSeen == nil {- status.State = swarmtypes.NodeAgentStatePending- return status+ identity := kit.FromPtr(runtime.identity)+ reportedNodeID := strings.TrimSpace(identity.SwarmNodeID)+ reportedHostname := strings.TrimSpace(identity.Hostname)+ state := func() swarmtypes.NodeAgentState {+ switch {+ // Only a live tunnel with a completed identity probe can authoritatively detect a node-ID mismatch.+ case runtime.connected && runtime.identity != nil:+ return kit.Ternary(identity.SwarmActive && reportedNodeID == strings.TrimSpace(nodeID), swarmtypes.NodeAgentStateConnected, swarmtypes.NodeAgentStateMismatched)+ // A fresh check-in proves the agent Arcane mapped to this node at deploy time is reachable.+ case live:+ return swarmtypes.NodeAgentStateConnected+ // No live evidence: an unpaired agent stays pending; one seen before is offline.+ case env.Status == string(environment.EnvironmentStatusPending) || env.LastSeen == nil:+ return swarmtypes.NodeAgentStatePending+ default:+ return swarmtypes.NodeAgentStateOffline+ }+ }()++ return swarmtypes.NodeAgentStatus{+ State: state,+ BindingKind: new(kit.Ternary(env.Hidden, swarmtypes.NodeAgentBindingKindDedicated, swarmtypes.NodeAgentBindingKindEnvironment)),+ EnvironmentID: &env.ID,+ EnvironmentName: &env.Name,+ EnvironmentType: new(kit.Ternary(env.IsEdge, "edge", "direct")),+ Connected: &live,+ LastHeartbeat: runtime.lastHeartbeat,+ LastPollAt: runtime.lastPollAt,+ ReportedNodeID: kit.Ternary(reportedNodeID == "", nil, &reportedNodeID),+ ReportedHostname: kit.Ternary(reportedHostname == "", nil, &reportedHostname),}-- status.State = swarmtypes.NodeAgentStateOffline- return status}func (s *Service) UpdateNode(ctx context.Context, nodeID string, req swarmtypes.NodeUpdateRequest) error {@@ -710,35 +628,28 @@ func (s *Service) UpdateNode(ctx context.Context, nodeID string, req swarmtypes.return err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- nodeResult, err := dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})+ nodeResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeInspectResult, error) {+ return dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})+ })if err != nil {return fmt.Errorf("failed to inspect swarm node: %w", err)}version := cmp.Or(req.Version, nodeResult.Node.Version.Index)- spec := nodeResult.Node.Spec- if req.Name != nil {- spec.Name = *req.Name- }- if req.Labels != nil {- spec.Labels = req.Labels- }- if req.Role != nil {- spec.Role = *req.Role- }- if req.Availability != nil {- spec.Availability = *req.Availability+ current := nodeResult.Node.Spec+ spec := swarm.NodeSpec{+ Name: mo.PointerToOption(req.Name).OrElse(current.Name),+ Labels: kit.Ternary(req.Labels != nil, req.Labels, current.Labels),+ Role: mo.PointerToOption(req.Role).OrElse(current.Role),+ Availability: mo.PointerToOption(req.Availability).OrElse(current.Availability),}- if _, nodeUpdateErr := dockerClient.NodeUpdate(ctx, nodeID, client.NodeUpdateOptions{- Version: swarm.Version{Index: version},- Spec: spec,+ if _, nodeUpdateErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeUpdateResult, error) {+ return dockerClient.NodeUpdate(ctx, nodeID, client.NodeUpdateOptions{+ Version: swarm.Version{Index: version},+ Spec: spec,+ })}); nodeUpdateErr != nil {return fmt.Errorf("failed to update swarm node: %w", nodeUpdateErr)}@@ -751,12 +662,9 @@ func (s *Service) RemoveNode(ctx context.Context, nodeID string, force bool) errreturn err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, nodeRemoveErr := dockerClient.NodeRemove(ctx, nodeID, client.NodeRemoveOptions{Force: force}); nodeRemoveErr != nil {+ if _, nodeRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeRemoveResult, error) {+ return dockerClient.NodeRemove(ctx, nodeID, client.NodeRemoveOptions{Force: force})+ }); nodeRemoveErr != nil {return fmt.Errorf("failed to remove swarm node: %w", nodeRemoveErr)}@@ -781,7 +689,7 @@ func (s *Service) ListNodeTasksPaginated(ctx context.Context, nodeID string, parreturn s.listTasks(ctx, filters, params)}-func (s *Service) buildNodePaginationConfigInternal() pagination.Config[swarmtypes.NodeSummary] {+func (s *Service) buildNodePaginationConfig() pagination.Config[swarmtypes.NodeSummary] {return pagination.Config[swarmtypes.NodeSummary]{SearchAccessors: []pagination.SearchAccessor[swarmtypes.NodeSummary]{func(node swarmtypes.NodeSummary) (string, error) { return node.Hostname, nil },@@ -808,7 +716,7 @@ func (s *Service) GetJoinCandidates(ctx context.Context, environmentID string) (if environmentID == environment.LocalEnvironmentID {identity, err = s.GetLocalNodeIdentity(ctx)} else {- identity, err = s.fetchSwarmNodeIdentityViaEdgeInternal(ctx, environmentID)+ identity, err = s.fetchSwarmNodeIdentityViaEdge(ctx, environmentID)}if err != nil {return nil, fmt.Errorf("failed to inspect Easy Join manager: %w", err)@@ -854,10 +762,10 @@ func (s *Service) GetJoinCandidates(ctx context.Context, environmentID string) (return candidates, nil}-// describeSwarmJoinFailureInternal turns a proxied join failure into a concise,+// describeSwarmJoinFailure turns a proxied join failure into a concise,// token-free message. Agent HTTP errors contribute their detail; transport// errors keep their own text.-func describeSwarmJoinFailureInternal(err error, joinToken string) string {+func describeSwarmJoinFailure(err error, joinToken string) string {var status *remenv.StatusErrorif !errors.As(err, &status) {return RedactSwarmJoinToken(err.Error(), joinToken)@@ -877,7 +785,7 @@ func describeSwarmJoinFailureInternal(err error, joinToken string) string {return fmt.Sprintf("swarm join failed with HTTP %d from the target agent", status.StatusCode)}-// redactSwarmJoinTokenInternal masks the join token wherever it appears in message.+// redactSwarmJoinToken masks the join token wherever it appears in message.func RedactSwarmJoinToken(message, joinToken string) string {if joinToken == "" {return message@@ -885,9 +793,9 @@ func RedactSwarmJoinToken(message, joinToken string) string {return strings.ReplaceAll(message, joinToken, "[redacted]")}-// selectSwarmManagerAddressesInternal prefers explicit manager addresses and+// selectSwarmManagerAddresses prefers explicit manager addresses and// otherwise uses the addresses advertised by the cluster's manager nodes.-func selectSwarmManagerAddressesInternal(explicit []string, nodes []swarmtypes.NodeSummary) ([]string, error) {+func selectSwarmManagerAddresses(explicit []string, nodes []swarmtypes.NodeSummary) ([]string, error) {addrs := kit.TrimNonEmpty(explicit)if len(addrs) > 0 {return addrs, nil@@ -903,7 +811,7 @@ func selectSwarmManagerAddressesInternal(explicit []string, nodes []swarmtypes.Nreturn addrs, nil}-func buildNodeAgentCandidatesInternal(environments []environment.Environment) []swarmtypes.NodeAgentCandidate {+func buildNodeAgentCandidates(environments []environment.Environment) []swarmtypes.NodeAgentCandidate {candidates := make([]swarmtypes.NodeAgentCandidate, 0, len(environments))for _, environment := range environments {environmentType := kit.Ternary(environment.IsEdge, "edge", "direct")@@ -915,16 +823,3 @@ func buildNodeAgentCandidatesInternal(environments []environment.Environment) []}return candidates}--func preferredNodeAgentEnvironmentInternal(environments []environment.Environment) environment.Environment {- var preferred environment.Environment- for index, environment := range environments {- if index == 0 {- preferred = environment- }- if !environment.Hidden {- return environment- }- }- return preferred-}diff --git a/backend/internal/swarm/children/nodes/service_test.go b/backend/internal/swarm/children/nodes/service_test.goindex d3a1d95d8..bb884523b 100644--- a/backend/internal/swarm/children/nodes/service_test.go+++ b/backend/internal/swarm/children/nodes/service_test.go@@ -21,7 +21,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/remenv")-func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {+func setupSwarmServiceTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -29,7 +29,7 @@ func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForSwarmTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()svc, err := settings.NewSettingsService(ctx, db)if err == nil {@@ -38,7 +38,7 @@ func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, dreturn svc, err}-func createSwarmTestEnvironmentInternal(t *testing.T, db *database.DB, id, apiURL, status string, isEdge bool, accessToken *string) {+func createSwarmTestEnvironment(t *testing.T, db *database.DB, id, apiURL, status string, isEdge bool, accessToken *string) {t.Helper()now := time.Now()require.NoError(t, db.Create(&environment.Environment{@@ -52,26 +52,26 @@ func createSwarmTestEnvironmentInternal(t *testing.T, db *database.DB, id, apiUR}).Error)}-func TestSelectSwarmManagerAddressesInternal(t *testing.T) {+func TestSelectSwarmManagerAddresses(t *testing.T) {nodes := []swarm.NodeSummary{{ID: "manager-1", ManagerAddress: "10.0.0.1:2377"},{ID: "worker-1"},{ID: "manager-2", ManagerAddress: "10.0.0.2:2377"},}- addrs, err := selectSwarmManagerAddressesInternal([]string{" 100.64.0.10:2377 ", "", "100.64.0.11:2377"}, nodes)+ addrs, err := selectSwarmManagerAddresses([]string{" 100.64.0.10:2377 ", "", "100.64.0.11:2377"}, nodes)require.NoError(t, err)require.Equal(t, []string{"100.64.0.10:2377", "100.64.0.11:2377"}, addrs)- addrs, err = selectSwarmManagerAddressesInternal([]string{" "}, nodes)+ addrs, err = selectSwarmManagerAddresses([]string{" "}, nodes)require.NoError(t, err)require.Equal(t, []string{"10.0.0.1:2377", "10.0.0.2:2377"}, addrs)- _, err = selectSwarmManagerAddressesInternal(nil, []swarm.NodeSummary{{ID: "worker-1"}})+ _, err = selectSwarmManagerAddresses(nil, []swarm.NodeSummary{{ID: "worker-1"}})require.ErrorIs(t, err, common.ErrBadRequest)}-func TestDescribeSwarmJoinFailureInternal(t *testing.T) {+func TestDescribeSwarmJoinFailure(t *testing.T) {const token = "SWMTKN-1-secret"statusErr := func(code int, body string) error {return errors.Join(&remenv.StatusError{StatusCode: code, Body: []byte(body)})@@ -79,7 +79,7 @@ func TestDescribeSwarmJoinFailureInternal(t *testing.T) {require.Equal(t,"failed to join swarm: Timeout was reached before node joined using [redacted]",- describeSwarmJoinFailureInternal(+ describeSwarmJoinFailure(statusErr(500,"{\"title\":\"Internal Server Error\",\"status\":500,\"detail\":\"failed to join swarm: Timeout was reached be"+@@ -88,20 +88,20 @@ func TestDescribeSwarmJoinFailureInternal(t *testing.T) {token,),)- require.Equal(t, "legacy join failure", describeSwarmJoinFailureInternal(statusErr(400, `{"success":false,"error":"legacy join failure"}`), token))- require.Equal(t, "swarm join failed with HTTP 502 from the target agent", describeSwarmJoinFailureInternal(statusErr(502, "<html>bad gateway</html>"), token))- require.Equal(t, "swarm join failed with HTTP 404 from the target agent", describeSwarmJoinFailureInternal(statusErr(404, ""), token))+ require.Equal(t, "legacy join failure", describeSwarmJoinFailure(statusErr(400, `{"success":false,"error":"legacy join failure"}`), token))+ require.Equal(t, "swarm join failed with HTTP 502 from the target agent", describeSwarmJoinFailure(statusErr(502, "<html>bad gateway</html>"), token))+ require.Equal(t, "swarm join failed with HTTP 404 from the target agent", describeSwarmJoinFailure(statusErr(404, ""), token))require.Equal(t,"failed to send request to environment edge: dial tcp 10.0.0.9:3552: connect: connection refused",- describeSwarmJoinFailureInternal(errors.New("failed to send request to environment edge: dial tcp 10.0.0.9:3552: connect: connection refused"), token),+ describeSwarmJoinFailure(errors.New("failed to send request to environment edge: dial tcp 10.0.0.9:3552: connect: connection refused"), token),)- require.Equal(t, "token [redacted] rejected", describeSwarmJoinFailureInternal(errors.New("token "+token+" rejected"), token))+ require.Equal(t, "token [redacted] rejected", describeSwarmJoinFailure(errors.New("token "+token+" rejected"), token))}-func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToken(t *testing.T) {+func TestService_FetchSwarmNodeIdentityViaEdge_UsesEnvironmentAccessToken(t *testing.T) {ctx := t.Context()- db := setupSwarmServiceTestDBInternal(t)- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)+ db := setupSwarmServiceTestDB(t)+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)require.NoError(t, err)envSvc := environment.NewEnvironmentService(db, nil, nil, nil, settingsSvc, nil)@@ -125,7 +125,7 @@ func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToke}))defer server.Close()- createSwarmTestEnvironmentInternal(+ createSwarmTestEnvironment(t,db,"env-1",@@ -137,7 +137,7 @@ func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessTokesvc := NewService(nil, nil, nil, nil, envSvc)- identity, err := svc.fetchSwarmNodeIdentityViaEdgeInternal(ctx, "env-1")+ identity, err := svc.fetchSwarmNodeIdentityViaEdge(ctx, "env-1")require.NoError(t, err)require.NotNil(t, identity)require.Equal(t, "node-1", identity.SwarmNodeID)@@ -151,7 +151,7 @@ func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToke// The regression case is a poll-mode agent: its persisted env.Status never leaves// "pending" (HandlePoll only updates the in-memory poll registry), so a fresh// lastPollAt must still resolve to "connected" rather than "pending".-func TestService_BuildNodeAgentStatusInternal(t *testing.T) {+func TestService_BuildNodeAgentStatus(t *testing.T) {const nodeID = "node-abc"now := time.Now()svc := &Service{}@@ -196,7 +196,7 @@ func TestService_BuildNodeAgentStatusInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := svc.buildNodeAgentStatusInternal(nodeID, tt.env, tt.runtime)+ got := svc.buildNodeAgentStatus(nodeID, tt.env, tt.runtime)require.Equal(t, tt.want, got.State)})}diff --git a/backend/internal/swarm/children/secrets/service.go b/backend/internal/swarm/children/secrets/service.goindex 24068afc5..ce8df0eeb 100644--- a/backend/internal/swarm/children/secrets/service.go+++ b/backend/internal/swarm/children/secrets/service.go@@ -2,7 +2,6 @@ package secretsimport ("context"- stdjson "encoding/json""encoding/json/v2""errors""fmt"@@ -11,16 +10,18 @@ import (swarmtypes "github.com/getarcaneapp/arcane/types/v2/swarm""github.com/moby/moby/api/types/swarm""github.com/moby/moby/client"++ "github.com/getarcaneapp/arcane/backend/v2/internal/docker")// Service manages swarm secrets.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)+ docker *docker.DockerClientServiceensureManager func(ctx context.Context) error}-func NewService(dockerClient func(ctx context.Context) (*client.Client, error), ensureManager func(ctx context.Context) error) *Service {- return &Service{dockerClient: dockerClient, ensureManager: ensureManager}+func NewService(dockerService *docker.DockerClientService, ensureManager func(ctx context.Context) error) *Service {+ return &Service{docker: dockerService, ensureManager: ensureManager}}func (s *Service) ListSecrets(ctx context.Context) ([]swarmtypes.SecretSummary, error) {@@ -28,12 +29,9 @@ func (s *Service) ListSecrets(ctx context.Context) ([]swarmtypes.SecretSummary,return nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- secretsResult, err := dockerClient.SecretList(ctx, client.SecretListOptions{})+ secretsResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretListResult, error) {+ return dockerClient.SecretList(ctx, client.SecretListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list swarm secrets: %w", err)}@@ -50,12 +48,9 @@ func (s *Service) GetSecret(ctx context.Context, secretID string) (*swarmtypes.Sreturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- secretResult, err := dockerClient.SecretInspect(ctx, secretID, client.SecretInspectOptions{})+ secretResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretInspectResult, error) {+ return dockerClient.SecretInspect(ctx, secretID, client.SecretInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm secret: %w", err)}@@ -68,17 +63,20 @@ func (s *Service) CreateSecret(ctx context.Context, req swarmtypes.SecretCreateRreturn nil, err}- spec, err := decodeSecretSpecInternal(req.Spec)- if err != nil {- return nil, err+ if trimmed := strings.TrimSpace(string(req.Spec)); trimmed == "" || trimmed == "null" {+ return nil, errors.New("secret spec is required")}-- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)+ var spec swarm.SecretSpec+ if err := json.Unmarshal(req.Spec, &spec); err != nil {+ return nil, fmt.Errorf("failed to parse secret spec: %w", err)+ }+ if strings.TrimSpace(spec.Name) == "" {+ return nil, errors.New("secret spec name is required")}- createResult, err := dockerClient.SecretCreate(ctx, client.SecretCreateOptions{Spec: spec})+ createResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretCreateResult, error) {+ return dockerClient.SecretCreate(ctx, client.SecretCreateOptions{Spec: spec})+ })if err != nil {return nil, fmt.Errorf("failed to create swarm secret: %w", err)}@@ -91,31 +89,11 @@ func (s *Service) RemoveSecret(ctx context.Context, secretID string) error {return err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, secretRemoveErr := dockerClient.SecretRemove(ctx, secretID, client.SecretRemoveOptions{}); secretRemoveErr != nil {+ if _, secretRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretRemoveResult, error) {+ return dockerClient.SecretRemove(ctx, secretID, client.SecretRemoveOptions{})+ }); secretRemoveErr != nil {return fmt.Errorf("failed to remove swarm secret: %w", secretRemoveErr)}return nil}--func decodeSecretSpecInternal(raw stdjson.RawMessage) (swarm.SecretSpec, error) {- if len(raw) == 0 || strings.TrimSpace(string(raw)) == "" || strings.TrimSpace(string(raw)) == "null" {- return swarm.SecretSpec{}, errors.New("secret spec is required")- }-- var spec swarm.SecretSpec- if err := json.Unmarshal(raw, &spec); err != nil {- return swarm.SecretSpec{}, fmt.Errorf("failed to parse secret spec: %w", err)- }-- if strings.TrimSpace(spec.Name) == "" {- return swarm.SecretSpec{}, errors.New("secret spec name is required")- }-- return spec, nil-}diff --git a/backend/internal/swarm/children/services/service.go b/backend/internal/swarm/children/services/service.goindex d0c27f51c..862cc88ec 100644--- a/backend/internal/swarm/children/services/service.go+++ b/backend/internal/swarm/children/services/service.go@@ -16,23 +16,25 @@ import ("github.com/moby/moby/api/types/swarm""github.com/moby/moby/client""go.getarcane.app/docker"+ "go.getarcane.app/kit/pkg"+ dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination")// Service manages swarm services.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)+ docker *dockerInternal.DockerClientServiceensureManager func(ctx context.Context) errorlistTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error)}func NewService(- dockerClient func(ctx context.Context) (*client.Client, error),+ dockerService *dockerInternal.DockerClientService,ensureManager func(ctx context.Context) error,listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error),) *Service {- return &Service{dockerClient: dockerClient, ensureManager: ensureManager, listTasks: listTasks}+ return &Service{docker: dockerService, ensureManager: ensureManager, listTasks: listTasks}}func (s *Service) ListServicesPaginated(ctx context.Context, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error) {@@ -40,32 +42,56 @@ func (s *Service) ListServicesPaginated(ctx context.Context, params pagination.Qreturn nil, pagination.Response{}, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{Status: true})+ servicesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{Status: true})+ })if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list swarm services: %w", err)}- return s.PaginateSummaries(ctx, dockerClient, servicesResult.Items, params)+ return s.PaginateSummaries(ctx, servicesResult.Items, params)}// PaginateSummaries summarizes services with their running nodes and networks, then paginates them.func (s *Service) PaginateSummaries(ctx context.Context,- dockerClient *client.Client,services []swarm.Service,params pagination.QueryParams,) ([]swarmtypes.ServiceSummary, pagination.Response, error) {- summaries, err := s.summarizeServicesInternal(ctx, dockerClient, services)+ summaries, err := s.summarizeServices(ctx, services)if err != nil {return nil, pagination.Response{}, err}- result := s.buildServicePaginationConfigInternal().SearchOrderAndPaginate(summaries, params)+ config := pagination.Config[swarmtypes.ServiceSummary]{+ SearchAccessors: []pagination.SearchAccessor[swarmtypes.ServiceSummary]{+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Name, nil },+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Image, nil },+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.ID, nil },+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.StackName, nil },+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Mode, nil },+ func(svc swarmtypes.ServiceSummary) (string, error) {+ return strings.Join(svc.Networks, " "), nil+ },+ func(svc swarmtypes.ServiceSummary) (string, error) {+ return strings.Join(svc.Nodes, " "), nil+ },+ },+ SortBindings: []pagination.SortBinding[swarmtypes.ServiceSummary]{+ {Key: "name", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Name, b.Name) }},+ {Key: "image", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Image, b.Image) }},+ {Key: "mode", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Mode, b.Mode) }},+ {Key: "replicas", Fn: func(a, b swarmtypes.ServiceSummary) int {+ if localCmp := cmp.Compare(a.Replicas, b.Replicas); localCmp != 0 {+ return localCmp+ }+ return cmp.Compare(a.RunningReplicas, b.RunningReplicas)+ }},+ {Key: "created", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},+ {Key: "updated", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},+ },+ }+ result := config.SearchOrderAndPaginate(summaries, params)return result.Items, pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params), nil}@@ -74,27 +100,26 @@ func (s *Service) GetService(ctx context.Context, serviceID string) (*swarmtypesreturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- serviceResult, err := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ serviceResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm service: %w", err)}service := serviceResult.Serviceinspect := swarmtypes.NewServiceInspect(service)- inspect.Nodes = s.resolveServiceNodeNamesInternal(ctx, dockerClient, serviceID)- inspect.NetworkDetails = s.enrichServiceNetworkDetailsInternal(ctx, dockerClient, service.Spec.TaskTemplate.Networks)- inspect.Mounts = s.enrichServiceMountsInternal(ctx, dockerClient, service.Spec.TaskTemplate.ContainerSpec)+ inspect.Nodes = s.resolveServiceNodeNames(ctx, serviceID)+ inspect.NetworkDetails = s.enrichServiceNetworkDetails(ctx, service.Spec.TaskTemplate.Networks)+ inspect.Mounts = s.enrichServiceMounts(ctx, service.Spec.TaskTemplate.ContainerSpec)return &inspect, nil}-func (s *Service) resolveServiceNodeNamesInternal(ctx context.Context, dockerClient *client.Client, serviceID string) []string {- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})+func (s *Service) resolveServiceNodeNames(ctx context.Context, serviceID string) []string {+ nodesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {+ return dockerClient.NodeList(ctx, client.NodeListOptions{})+ })if err != nil {return nil}@@ -104,7 +129,9 @@ func (s *Service) resolveServiceNodeNamesInternal(ctx context.Context, dockerClinodeNameByID[node.ID] = node.Description.Hostname}- tasksResult, err := dockerClient.TaskList(ctx, client.TaskListOptions{})+ tasksResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {+ return dockerClient.TaskList(ctx, client.TaskListOptions{})+ })if err != nil {return nil}@@ -123,9 +150,8 @@ func (s *Service) resolveServiceNodeNamesInternal(ctx context.Context, dockerClireturn nodeNames}-func (s *Service) enrichServiceNetworkDetailsInternal(+func (s *Service) enrichServiceNetworkDetails(ctx context.Context,- dockerClient *client.Client,networkConfigs []swarm.NetworkAttachmentConfig,) map[string]swarmtypes.ServiceNetworkDetail {if len(networkConfigs) == 0 {@@ -135,7 +161,9 @@ func (s *Service) enrichServiceNetworkDetailsInternal(details := make(map[string]swarmtypes.ServiceNetworkDetail, len(networkConfigs))for _, networkConfig := range networkConfigs {networkID := networkConfig.Target- netInspectResult, err := dockerClient.NetworkInspect(ctx, networkID, client.NetworkInspectOptions{})+ netInspectResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {+ return dockerClient.NetworkInspect(ctx, networkID, client.NetworkInspectOptions{})+ })if err != nil {continue}@@ -153,15 +181,17 @@ func (s *Service) enrichServiceNetworkDetailsInternal(EnableIPv6: networkInfo.EnableIPv6,ConfigOnly: networkInfo.ConfigOnly,Options: networkInfo.Options,+ ConfigFrom: networkInfo.ConfigFrom.Network,}- detail.ConfigFrom = cmp.Or(networkInfo.ConfigFrom.Network, detail.ConfigFrom)for _, ipamCfg := range networkInfo.IPAM.Config {- detail.IPAMConfigs = append(detail.IPAMConfigs, toServiceNetworkIPAMConfigInternal(ipamCfg))+ detail.IPAMConfigs = append(detail.IPAMConfigs, toServiceNetworkIPAMConfig(ipamCfg))}if detail.ConfigFrom != "" {- configInspectResult, networkInspectErr := dockerClient.NetworkInspect(ctx, detail.ConfigFrom, client.NetworkInspectOptions{})+ configInspectResult, networkInspectErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {+ return dockerClient.NetworkInspect(ctx, detail.ConfigFrom, client.NetworkInspectOptions{})+ })if networkInspectErr == nil {configNetwork := configInspectResult.NetworkconfigDetail := &swarmtypes.ServiceNetworkConfigDetail{@@ -173,7 +203,7 @@ func (s *Service) enrichServiceNetworkDetailsInternal(Options: configNetwork.Options,}for _, ipamCfg := range configNetwork.IPAM.Config {- converted := toServiceNetworkIPAMConfigInternal(ipamCfg)+ converted := toServiceNetworkIPAMConfig(ipamCfg)if ipamCfg.Subnet.IsValid() && ipamCfg.Subnet.Addr().Is6() {configDetail.IPv6Configs = append(configDetail.IPv6Configs, converted)continue@@ -190,9 +220,8 @@ func (s *Service) enrichServiceNetworkDetailsInternal(return details}-func (s *Service) enrichServiceMountsInternal(+func (s *Service) enrichServiceMounts(ctx context.Context,- dockerClient *client.Client,containerSpec *swarm.ContainerSpec,) []swarmtypes.ServiceMount {if containerSpec == nil {@@ -201,22 +230,27 @@ func (s *Service) enrichServiceMountsInternal(mounts := make([]swarmtypes.ServiceMount, 0, len(containerSpec.Mounts))for _, serviceMount := range containerSpec.Mounts {- mount := swarmtypes.ServiceMount{- Type: string(serviceMount.Type),- Source: serviceMount.Source,- Target: serviceMount.Target,- ReadOnly: serviceMount.ReadOnly,- }- if serviceMount.Type == "volume" && serviceMount.Source != "" {- volInspectResult, err := dockerClient.VolumeInspect(ctx, serviceMount.Source, client.VolumeInspectOptions{})- if err == nil {- volume := volInspectResult.Volume- mount.VolumeDriver = volume.Driver- mount.VolumeOptions = volume.Options- mount.DevicePath = volume.Mountpoint+ volumeInfo := func() client.VolumeInspectResult {+ if serviceMount.Type != "volume" || serviceMount.Source == "" {+ return client.VolumeInspectResult{}}- }- mounts = append(mounts, mount)+ result, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, serviceMount.Source, client.VolumeInspectOptions{})+ })+ if err != nil {+ return client.VolumeInspectResult{}+ }+ return result+ }().Volume+ mounts = append(mounts, swarmtypes.ServiceMount{+ Type: string(serviceMount.Type),+ Source: serviceMount.Source,+ Target: serviceMount.Target,+ ReadOnly: serviceMount.ReadOnly,+ VolumeDriver: volumeInfo.Driver,+ VolumeOptions: volumeInfo.Options,+ DevicePath: volumeInfo.Mountpoint,+ })}return mounts@@ -227,11 +261,6 @@ func (s *Service) CreateService(ctx context.Context, req swarmtypes.ServiceCreatreturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-// Unmarshal spec from JSONvar spec swarm.ServiceSpecif unmarshalErr := json.Unmarshal(req.Spec, &spec); unmarshalErr != nil {@@ -244,12 +273,14 @@ func (s *Service) CreateService(ctx context.Context, req swarmtypes.ServiceCreat}// Sanitize spec to avoid empty UID/GID in secret/config refs- sanitizeServiceSpecInternal(&spec)-- resp, err := dockerClient.ServiceCreate(ctx, client.ServiceCreateOptions{- Spec: spec,- EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,- QueryRegistry: optionsPayload.QueryRegistry,+ sanitizeServiceSpec(&spec)++ resp, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceCreateResult, error) {+ return dockerClient.ServiceCreate(ctx, client.ServiceCreateOptions{+ Spec: spec,+ EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,+ QueryRegistry: optionsPayload.QueryRegistry,+ })})if err != nil {return nil, fmt.Errorf("failed to create swarm service: %w", err)@@ -266,14 +297,11 @@ func (s *Service) UpdateService(ctx context.Context, serviceID string, req swarmreturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-versionIndex := req.Versionif versionIndex == 0 {- serviceResult, serviceInspectErr := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ serviceResult, serviceInspectErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ })if serviceInspectErr != nil {return nil, fmt.Errorf("failed to inspect swarm service: %w", serviceInspectErr)}@@ -286,15 +314,17 @@ func (s *Service) UpdateService(ctx context.Context, serviceID string, req swarm}// Sanitize spec to avoid empty UID/GID in secret/config refs- sanitizeServiceSpecInternal(&req.Spec)-- resp, err := dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{- Version: swarm.Version{Index: versionIndex},- Spec: req.Spec,- EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,- RegistryAuthFrom: optionsPayload.RegistryAuthFrom,- Rollback: optionsPayload.Rollback,- QueryRegistry: optionsPayload.QueryRegistry,+ sanitizeServiceSpec(&req.Spec)++ resp, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceUpdateResult, error) {+ return dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{+ Version: swarm.Version{Index: versionIndex},+ Spec: req.Spec,+ EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,+ RegistryAuthFrom: optionsPayload.RegistryAuthFrom,+ Rollback: optionsPayload.Rollback,+ QueryRegistry: optionsPayload.QueryRegistry,+ })})if err != nil {return nil, fmt.Errorf("failed to update swarm service: %w", err)@@ -310,12 +340,9 @@ func (s *Service) RemoveService(ctx context.Context, serviceID string) error {return err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, serviceRemoveErr := dockerClient.ServiceRemove(ctx, serviceID, client.ServiceRemoveOptions{}); serviceRemoveErr != nil {+ if _, serviceRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceRemoveResult, error) {+ return dockerClient.ServiceRemove(ctx, serviceID, client.ServiceRemoveOptions{})+ }); serviceRemoveErr != nil {return fmt.Errorf("failed to remove swarm service: %w", serviceRemoveErr)}@@ -327,7 +354,7 @@ func (s *Service) StreamServiceLogs(ctx context.Context, serviceID string, logsCreturn err}- dockerClient, err := s.dockerClient(ctx)+ dockerClient, err := s.docker.GetClient(ctx)if err != nil {return fmt.Errorf("failed to connect to Docker: %w", err)}@@ -370,20 +397,19 @@ func (s *Service) RollbackService(ctx context.Context, serviceID string) (*swarmreturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- serviceResult, err := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ serviceResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm service: %w", err)}- updateResult, err := dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{- Version: serviceResult.Service.Version,- Spec: serviceResult.Service.Spec,- Rollback: "previous",+ updateResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceUpdateResult, error) {+ return dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{+ Version: serviceResult.Service.Version,+ Spec: serviceResult.Service.Spec,+ Rollback: "previous",+ })})if err != nil {return nil, fmt.Errorf("failed to rollback swarm service: %w", err)@@ -397,24 +423,28 @@ func (s *Service) ScaleService(ctx context.Context, serviceID string, replicas ureturn nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- serviceResult, err := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ serviceResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm service: %w", err)}service := serviceResult.Service- if applySwarmServiceScaleErr := applySwarmServiceScaleInternal(&service.Spec.Mode, replicas); applySwarmServiceScaleErr != nil {- return nil, applySwarmServiceScaleErr+ switch {+ case service.Spec.Mode.Replicated != nil:+ service.Spec.Mode.Replicated.Replicas = &replicas+ case service.Spec.Mode.ReplicatedJob != nil:+ service.Spec.Mode.ReplicatedJob.TotalCompletions = &replicas+ default:+ return nil, fmt.Errorf("scale can only be used with replicated or replicated-job mode: %w", errdefs.ErrInvalidArgument)}- updateResult, err := dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{- Version: service.Version,- Spec: service.Spec,+ updateResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceUpdateResult, error) {+ return dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{+ Version: service.Version,+ Spec: service.Spec,+ })})if err != nil {return nil, fmt.Errorf("failed to scale swarm service: %w", err)@@ -423,8 +453,10 @@ func (s *Service) ScaleService(ctx context.Context, serviceID string, replicas ureturn &swarmtypes.ServiceUpdateResponse{Warnings: updateResult.Warnings}, nil}-func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *client.Client, services []swarm.Service) ([]swarmtypes.ServiceSummary, error) {- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})+func (s *Service) summarizeServices(ctx context.Context, services []swarm.Service) ([]swarmtypes.ServiceSummary, error) {+ nodesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {+ return dockerClient.NodeList(ctx, client.NodeListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list swarm nodes: %w", err)}@@ -434,7 +466,9 @@ func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *cnodeNameByID[node.ID] = node.Description.Hostname}- networksResult, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})+ networksResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkListResult, error) {+ return dockerClient.NetworkList(ctx, client.NetworkListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list networks: %w", err)}@@ -449,7 +483,9 @@ func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *cserviceIDs[service.ID] = struct{}{}}- tasksResult, err := dockerClient.TaskList(ctx, client.TaskListOptions{})+ tasksResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {+ return dockerClient.TaskList(ctx, client.TaskListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list swarm tasks: %w", err)}@@ -486,88 +522,34 @@ func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *creturn summaries, nil}-func (s *Service) buildServicePaginationConfigInternal() pagination.Config[swarmtypes.ServiceSummary] {- return pagination.Config[swarmtypes.ServiceSummary]{- SearchAccessors: []pagination.SearchAccessor[swarmtypes.ServiceSummary]{- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Name, nil },- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Image, nil },- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.ID, nil },- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.StackName, nil },- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Mode, nil },- func(svc swarmtypes.ServiceSummary) (string, error) {- return strings.Join(svc.Networks, " "), nil- },- func(svc swarmtypes.ServiceSummary) (string, error) {- return strings.Join(svc.Nodes, " "), nil- },- },- SortBindings: []pagination.SortBinding[swarmtypes.ServiceSummary]{- {Key: "name", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Name, b.Name) }},- {Key: "image", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Image, b.Image) }},- {Key: "mode", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Mode, b.Mode) }},- {Key: "replicas", Fn: func(a, b swarmtypes.ServiceSummary) int {- if localCmp := cmp.Compare(a.Replicas, b.Replicas); localCmp != 0 {- return localCmp- }- return cmp.Compare(a.RunningReplicas, b.RunningReplicas)- }},- {Key: "created", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},- {Key: "updated", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},- },- }-}--func sanitizeServiceSpecInternal(spec *swarm.ServiceSpec) {+func sanitizeServiceSpec(spec *swarm.ServiceSpec) {if spec == nil || spec.TaskTemplate.ContainerSpec == nil {return}-+ // Secret and config file targets default to root ownership.+ defaultOwner := func(ids ...*string) {+ for _, id := range ids {+ if strings.TrimSpace(*id) == "" {+ *id = "0"+ }+ }+ }for _, ref := range spec.TaskTemplate.ContainerSpec.Secrets {if ref != nil && ref.File != nil {- if strings.TrimSpace(ref.File.UID) == "" {- ref.File.UID = "0"- }- if strings.TrimSpace(ref.File.GID) == "" {- ref.File.GID = "0"- }+ defaultOwner(&ref.File.UID, &ref.File.GID)}}-for _, ref := range spec.TaskTemplate.ContainerSpec.Configs {if ref != nil && ref.File != nil {- if strings.TrimSpace(ref.File.UID) == "" {- ref.File.UID = "0"- }- if strings.TrimSpace(ref.File.GID) == "" {- ref.File.GID = "0"- }+ defaultOwner(&ref.File.UID, &ref.File.GID)}}}-func toServiceNetworkIPAMConfigInternal(cfg network.IPAMConfig) swarmtypes.ServiceNetworkIPAMConfig {- out := swarmtypes.ServiceNetworkIPAMConfig{}- if cfg.Subnet.IsValid() {- out.Subnet = cfg.Subnet.String()+func toServiceNetworkIPAMConfig(cfg network.IPAMConfig) swarmtypes.ServiceNetworkIPAMConfig {+ return swarmtypes.ServiceNetworkIPAMConfig{+ Subnet: kit.Ternary(cfg.Subnet.IsValid(), cfg.Subnet.String(), ""),+ Gateway: kit.Ternary(cfg.Gateway.IsValid(), cfg.Gateway.String(), ""),+ IPRange: kit.Ternary(cfg.IPRange.IsValid(), cfg.IPRange.String(), ""),}- if cfg.Gateway.IsValid() {- out.Gateway = cfg.Gateway.String()- }- if cfg.IPRange.IsValid() {- out.IPRange = cfg.IPRange.String()- }- return out-}--func applySwarmServiceScaleInternal(mode *swarm.ServiceMode, replicas uint64) error {- switch {- case mode.Replicated != nil:- mode.Replicated.Replicas = &replicas- case mode.ReplicatedJob != nil:- mode.ReplicatedJob.TotalCompletions = &replicas- default:- return fmt.Errorf("scale can only be used with replicated or replicated-job mode: %w", errdefs.ErrInvalidArgument)- }-- return nil}diff --git a/backend/internal/swarm/children/services/service_test.go b/backend/internal/swarm/children/services/service_test.goindex b9b7cbf74..32553e80e 100644--- a/backend/internal/swarm/children/services/service_test.go+++ b/backend/internal/swarm/children/services/service_test.go@@ -17,7 +17,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/docker")-func newSwarmTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {+func newSwarmTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {t.Helper()cli, err := client.New(client.WithHost(server.URL),@@ -29,7 +29,7 @@ func newSwarmTestDockerClientInternal(t *testing.T, server *httptest.Server) *clreturn cli}-func TestService_ScaleService_HandlesServiceModesInternal(t *testing.T) {+func TestService_ScaleService_HandlesServiceModes(t *testing.T) {ctx := t.Context()replicas := uint64(5)maxConcurrent := uint64(2)@@ -122,8 +122,8 @@ func TestService_ScaleService_HandlesServiceModesInternal(t *testing.T) {}))t.Cleanup(server.Close)- dockerService := &docker.DockerClientService{Client: newSwarmTestDockerClientInternal(t, server)}- svc := NewService(dockerService.GetClient, func(context.Context) error { return nil }, nil)+ dockerService := &docker.DockerClientService{Client: newSwarmTestDockerClient(t, server)}+ svc := NewService(dockerService, func(context.Context) error { return nil }, nil)resp, err := svc.ScaleService(ctx, "service-1", replicas)if tt.wantErr {diff --git a/backend/internal/swarm/children/stacks/service.go b/backend/internal/swarm/children/stacks/service.goindex b1b966deb..3dd1fdcb9 100644--- a/backend/internal/swarm/children/stacks/service.go+++ b/backend/internal/swarm/children/stacks/service.go@@ -9,6 +9,7 @@ import ("log/slog""path""path/filepath"+ "slices""strings""time"@@ -18,9 +19,9 @@ import ("github.com/moby/moby/client""go.getarcane.app/acfs""go.getarcane.app/acfs/types"- "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/common"+ dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/settings"libswarm "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/swarm""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"@@ -37,24 +38,24 @@ const (// Service deploys and manages swarm stacks and their persisted sources.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)+ docker *dockerInternal.DockerClientServiceensureManager func(ctx context.Context) errorlistTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error)- paginateServices func(ctx context.Context, dockerClient *client.Client, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error)+ paginateServices func(ctx context.Context, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error)registryAuth func(ctx context.Context, imageRef string) (string, error)settingsService *settings.SettingsService}func NewService(- dockerClient func(ctx context.Context) (*client.Client, error),+ dockerService *dockerInternal.DockerClientService,ensureManager func(ctx context.Context) error,listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error),- paginateServices func(ctx context.Context, dockerClient *client.Client, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error),+ paginateServices func(ctx context.Context, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error),registryAuth func(ctx context.Context, imageRef string) (string, error),settingsService *settings.SettingsService,) *Service {return &Service{- dockerClient: dockerClient,+ docker: dockerService,ensureManager: ensureManager,listTasks: listTasks,paginateServices: paginateServices,@@ -68,12 +69,9 @@ func (s *Service) ListStacksPaginated(ctx context.Context, environmentID string,return nil, pagination.Response{}, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{})+ servicesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{})+ })if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list swarm services: %w", err)}@@ -108,7 +106,7 @@ func (s *Service) ListStacksPaginated(ctx context.Context, environmentID string,}}- persistedStacks, err := s.listPersistedStackSourcesInternal(ctx, environmentID)+ persistedStacks, err := s.listPersistedStackSources(ctx, environmentID)if err != nil {return nil, pagination.Response{}, err}@@ -124,7 +122,18 @@ func (s *Service) ListStacksPaginated(ctx context.Context, environmentID string,items = append(items, *stack)}- config := s.buildStackPaginationConfigInternal()+ config := pagination.Config[swarmtypes.StackSummary]{+ SearchAccessors: []pagination.SearchAccessor[swarmtypes.StackSummary]{+ func(stack swarmtypes.StackSummary) (string, error) { return stack.Name, nil },+ func(stack swarmtypes.StackSummary) (string, error) { return stack.Namespace, nil },+ },+ SortBindings: []pagination.SortBinding[swarmtypes.StackSummary]{+ {Key: "name", Fn: func(a, b swarmtypes.StackSummary) int { return strings.Compare(a.Name, b.Name) }},+ {Key: "services", Fn: func(a, b swarmtypes.StackSummary) int { return cmp.Compare(a.Services, b.Services) }},+ {Key: "created", Fn: func(a, b swarmtypes.StackSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},+ {Key: "updated", Fn: func(a, b swarmtypes.StackSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},+ },+ }result := config.SearchOrderAndPaginate(items, params)paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params)@@ -141,18 +150,18 @@ func (s *Service) DeployStack(ctx context.Context, environmentID string, req swareturn nil, errors.New("stack name is required")}- dockerClient, err := s.dockerClient(ctx)+ dockerClient, err := s.docker.GetClient(ctx)if err != nil {return nil, fmt.Errorf("failed to connect to Docker: %w", err)}- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)if err != nil {return nil, err}- if upsertStackSourceErr := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {- slog.WarnContext(ctx, "failed to persist swarm stack source", "environmentId", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", upsertStackSourceErr)+ if upsertStackSourceErr := s.upsertStackSource(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {+ slog.WarnContext(ctx, "failed to persist swarm stack source", "environmentId", normalizeSwarmEnvironmentID(environmentID), "stackName", stackName, "error", upsertStackSourceErr)}workingDir := req.WorkingDir@@ -184,7 +193,7 @@ func (s *Service) DeployStack(ctx context.Context, environmentID string, req swaWorkingDir: workingDir,PathMapper: pm,}); deployStackErr != nil {- return nil, classifySwarmStackErrorInternal(deployStackErr)+ return nil, classifySwarmStackError(deployStackErr)}return &swarmtypes.StackDeployResponse{Name: stackName}, nil@@ -195,19 +204,14 @@ func (s *Service) GetStack(ctx context.Context, environmentID, stackName string)return nil, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)+ services, err := s.listStackServicesRaw(ctx, stackName)if err != nil {return nil, err}if len(services) == 0 {- persisted, getPersistedStackSourceSummaryErr := s.getPersistedStackSourceSummaryInternal(ctx, environmentID, stackName)- if getPersistedStackSourceSummaryErr != nil {- return nil, kit.Ternary[error](errdefs.IsNotFound(getPersistedStackSourceSummaryErr), errdefs.ErrNotFound, getPersistedStackSourceSummaryErr)+ persisted, persistedErr := s.getPersistedStackSourceSummary(ctx, environmentID, stackName)+ if persistedErr != nil {+ return nil, persistedErr}return &swarmtypes.StackInspect{@@ -245,7 +249,7 @@ func (s *Service) GetStackSource(ctx context.Context, environmentID, stackName sreturn nil, errors.New("stack name is required")}- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)if err != nil {return nil, err}@@ -306,7 +310,7 @@ func (s *Service) GetStackSource(ctx context.Context, environmentID, stackName s}, nil}-var deployStackAfterSourceUpdateInternal = (*Service).DeployStack+var deployStackAfterSourceUpdate = (*Service).DeployStackfunc (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackName string, req swarmtypes.StackSourceUpdateRequest) (*swarmtypes.StackSource, error) {stackName = strings.TrimSpace(stackName)@@ -322,7 +326,7 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNamreturn nil, err}- if upsertStackSourceErr := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {+ if upsertStackSourceErr := s.upsertStackSource(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {return nil, upsertStackSourceErr}@@ -334,7 +338,7 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam// resolution is per image and yields nothing unless a configured container// registry matches the image host. Without it, a service added in the edit// has no previous spec to fall back on and private images fail to pull.- if _, deployStackAfterSourceUpdateErr := deployStackAfterSourceUpdateInternal(s, ctx, environmentID, swarmtypes.StackDeployRequest{+ if _, deployStackAfterSourceUpdateErr := deployStackAfterSourceUpdate(s, ctx, environmentID, swarmtypes.StackDeployRequest{Name: stackName,ComposeContent: req.ComposeContent,OverrideContent: req.OverrideContent,@@ -347,12 +351,12 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam// was never successfully deployed.var restoreErr errorif previous != nil {- restoreErr = s.upsertStackSourceInternal(ctx, environmentID, stackName, previous.ComposeContent, previous.OverrideContent, previous.EnvContent, previous.Files)+ restoreErr = s.upsertStackSource(ctx, environmentID, stackName, previous.ComposeContent, previous.OverrideContent, previous.EnvContent, previous.Files)} else {- restoreErr = s.deleteStackSourceInternal(ctx, environmentID, stackName)+ restoreErr = s.deleteStackSource(ctx, environmentID, stackName)}if restoreErr != nil {- slog.WarnContext(ctx, "failed to restore swarm stack source after deploy failure", "environmentId", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", restoreErr)+ slog.WarnContext(ctx, "failed to restore swarm stack source after deploy failure", "environmentId", normalizeSwarmEnvironmentID(environmentID), "stackName", stackName, "error", restoreErr)}return nil, fmt.Errorf("failed to redeploy swarm stack from updated source: %w", deployStackAfterSourceUpdateErr)}@@ -366,8 +370,8 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam}, nil}-func (s *Service) listPersistedStackSourcesInternal(ctx context.Context, environmentID string) (map[string]swarmtypes.StackSummary, error) {- _, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDirInternal(ctx, environmentID)+func (s *Service) listPersistedStackSources(ctx context.Context, environmentID string) (map[string]swarmtypes.StackSummary, error) {+ _, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDir(ctx, environmentID)if err != nil {return nil, err}@@ -386,7 +390,7 @@ func (s *Service) listPersistedStackSourcesInternal(ctx context.Context, environcontinue}- summary, buildPersistedStackSourceSummaryErr := s.buildPersistedStackSourceSummaryInternal(ctx, filepath.Join(environmentDir, entry.Name), entry.Name)+ summary, buildPersistedStackSourceSummaryErr := s.buildPersistedStackSourceSummary(ctx, filepath.Join(environmentDir, entry.Name), entry.Name)if buildPersistedStackSourceSummaryErr != nil {if errdefs.IsNotFound(buildPersistedStackSourceSummaryErr) {continue@@ -400,16 +404,21 @@ func (s *Service) listPersistedStackSourcesInternal(ctx context.Context, environreturn stacks, nil}-func (s *Service) getPersistedStackSourceSummaryInternal(ctx context.Context, environmentID, stackName string) (*swarmtypes.StackSummary, error) {- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)+// getPersistedStackSourceSummary maps any not-found failure to errdefs.ErrNotFound.+func (s *Service) getPersistedStackSourceSummary(ctx context.Context, environmentID, stackName string) (*swarmtypes.StackSummary, error) {+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)if err != nil {return nil, err}- return s.buildPersistedStackSourceSummaryInternal(ctx, stackSourceDir, stackName)+ summary, err := s.buildPersistedStackSourceSummary(ctx, stackSourceDir, stackName)+ if errdefs.IsNotFound(err) {+ return nil, errdefs.ErrNotFound+ }+ return summary, err}-func (s *Service) buildPersistedStackSourceSummaryInternal(ctx context.Context, stackSourceDir, stackName string) (*swarmtypes.StackSummary, error) {+func (s *Service) buildPersistedStackSourceSummary(ctx context.Context, stackSourceDir, stackName string) (*swarmtypes.StackSummary, error) {composeEntry, err := acfs.Stat(ctx, stackSourceDir, "/"+swarmStackComposeFilename, true)if err != nil {if errors.Is(err, fs.ErrNotExist) {@@ -453,46 +462,48 @@ func (s *Service) RemoveStack(ctx context.Context, environmentID, stackName strireturn errors.New("stack name is required")}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)+ services, err := s.listStackServicesRaw(ctx, stackName)if err != nil {return err}if len(services) == 0 {- if _, getPersistedStackSourceSummaryErr := s.getPersistedStackSourceSummaryInternal(ctx, environmentID, stackName); getPersistedStackSourceSummaryErr != nil {- return kit.Ternary[error](errdefs.IsNotFound(getPersistedStackSourceSummaryErr), errdefs.ErrNotFound, getPersistedStackSourceSummaryErr)+ if _, persistedErr := s.getPersistedStackSourceSummary(ctx, environmentID, stackName); persistedErr != nil {+ return persistedErr}} else {- if removeStackServicesErr := s.removeStackServicesInternal(ctx, dockerClient, services); removeStackServicesErr != nil {+ if removeStackServicesErr := s.removeStackServices(ctx, services); removeStackServicesErr != nil {return removeStackServicesErr}}+ dockerClient, err := s.docker.GetClient(ctx)+ if err != nil {+ return fmt.Errorf("failed to connect to Docker: %w", err)+ }+if removeStackResourcesErr := libswarm.RemoveStackResources(ctx, dockerClient, stackName); removeStackResourcesErr != nil {- return classifySwarmStackErrorInternal(removeStackResourcesErr)+ return classifySwarmStackError(removeStackResourcesErr)}- if deleteStackSourceErr := s.deleteStackSourceInternal(ctx, environmentID, stackName); deleteStackSourceErr != nil {- slog.WarnContext(ctx, "failed to remove persisted swarm stack source", "environmentId", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", deleteStackSourceErr)+ if deleteStackSourceErr := s.deleteStackSource(ctx, environmentID, stackName); deleteStackSourceErr != nil {+ slog.WarnContext(ctx, "failed to remove persisted swarm stack source", "environmentId", normalizeSwarmEnvironmentID(environmentID), "stackName", stackName, "error", deleteStackSourceErr)}return nil}-func (s *Service) removeStackServicesInternal(ctx context.Context, dockerClient *client.Client, services []swarm.Service) error {+func (s *Service) removeStackServices(ctx context.Context, services []swarm.Service) error {serviceIDs := make(map[string]struct{}, len(services))for _, service := range services {serviceIDs[service.ID] = struct{}{}- if _, err := dockerClient.ServiceRemove(ctx, service.ID, client.ServiceRemoveOptions{}); err != nil && !errdefs.IsNotFound(err) {+ if _, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceRemoveResult, error) {+ return dockerClient.ServiceRemove(ctx, service.ID, client.ServiceRemoveOptions{})+ }); err != nil && !errdefs.IsNotFound(err) {return fmt.Errorf("failed to remove swarm service %s: %w", service.Spec.Name, err)}}- if err := s.waitForRemovedServiceTasksInternal(ctx, dockerClient, serviceIDs, 30*time.Second); err != nil {+ if err := s.waitForRemovedServiceTasks(ctx, serviceIDs, 30*time.Second); err != nil {return err}@@ -504,12 +515,7 @@ func (s *Service) ListStackServicesPaginated(ctx context.Context, stackName strireturn nil, pagination.Response{}, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)+ services, err := s.listStackServicesRaw(ctx, stackName)if err != nil {return nil, pagination.Response{}, err}@@ -517,7 +523,7 @@ func (s *Service) ListStackServicesPaginated(ctx context.Context, stackName strireturn nil, pagination.Response{}, errdefs.ErrNotFound}- return s.paginateServices(ctx, dockerClient, services, params)+ return s.paginateServices(ctx, services, params)}func (s *Service) ListStackTasksPaginated(ctx context.Context, stackName string, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {@@ -525,12 +531,7 @@ func (s *Service) ListStackTasksPaginated(ctx context.Context, stackName string,return nil, pagination.Response{}, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)+ services, err := s.listStackServicesRaw(ctx, stackName)if err != nil {return nil, pagination.Response{}, err}@@ -551,11 +552,11 @@ func (s *Service) RenderStackConfig(ctx context.Context, environmentID string, rreturn nil, err}- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, req.Name)+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, req.Name)if err != nil {return nil, err}- dockerClient, err := s.dockerClient(ctx)+ dockerClient, err := s.docker.GetClient(ctx)if err != nil {return nil, fmt.Errorf("failed to connect to Docker: %w", err)}@@ -575,7 +576,7 @@ func (s *Service) RenderStackConfig(ctx context.Context, environmentID string, rPathMapper: pm,})if err != nil {- return nil, classifySwarmStackErrorInternal(err)+ return nil, classifySwarmStackError(err)}return &swarmtypes.StackRenderConfigResponse{@@ -589,16 +590,18 @@ func (s *Service) RenderStackConfig(ctx context.Context, environmentID string, r}, nil}-func (s *Service) listStackServicesRawInternal(ctx context.Context, dockerClient *client.Client, stackName string) ([]swarm.Service, error) {+func (s *Service) listStackServicesRaw(ctx context.Context, stackName string) ([]swarm.Service, error) {stackName = strings.TrimSpace(stackName)if stackName == "" {return nil, errors.New("stack name is required")}stackFilter := make(client.Filters).Add("label", fmt.Sprintf("%s=%s", swarmtypes.StackNamespaceLabel, stackName))- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{- Filters: stackFilter,- Status: true,+ servicesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{+ Filters: stackFilter,+ Status: true,+ })})if err != nil {return nil, fmt.Errorf("failed to list stack services: %w", err)@@ -607,7 +610,7 @@ func (s *Service) listStackServicesRawInternal(ctx context.Context, dockerClientreturn servicesResult.Items, nil}-func (s *Service) waitForRemovedServiceTasksInternal(ctx context.Context, dockerClient *client.Client, serviceIDs map[string]struct{}, timeout time.Duration) error {+func (s *Service) waitForRemovedServiceTasks(ctx context.Context, serviceIDs map[string]struct{}, timeout time.Duration) error {if len(serviceIDs) == 0 {return nil}@@ -624,18 +627,23 @@ func (s *Service) waitForRemovedServiceTasksInternal(ctx context.Context, docker}for {- tasksResult, err := dockerClient.TaskList(waitCtx, client.TaskListOptions{Filters: taskFilters})+ tasksResult, err := s.docker.Call(waitCtx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {+ return dockerClient.TaskList(ctx, client.TaskListOptions{Filters: taskFilters})+ })if err != nil {return fmt.Errorf("failed to list tasks while waiting for stack removal: %w", err)}- hasActiveTasks := false- for _, task := range tasksResult.Items {- if !isTaskTerminalInternal(task.Status.State) {- hasActiveTasks = true- break- }- }+ hasActiveTasks := slices.ContainsFunc(tasksResult.Items, func(task swarm.Task) bool {+ return !slices.Contains([]swarm.TaskState{+ swarm.TaskStateComplete,+ swarm.TaskStateShutdown,+ swarm.TaskStateFailed,+ swarm.TaskStateRejected,+ swarm.TaskStateRemove,+ swarm.TaskStateOrphaned,+ }, task.Status.State)+ })if !hasActiveTasks {return nil}@@ -648,37 +656,13 @@ func (s *Service) waitForRemovedServiceTasksInternal(ctx context.Context, docker}}-func isTaskTerminalInternal(state swarm.TaskState) bool {- switch state {- case swarm.TaskStateComplete,- swarm.TaskStateShutdown,- swarm.TaskStateFailed,- swarm.TaskStateRejected,- swarm.TaskStateRemove,- swarm.TaskStateOrphaned:- return true- case swarm.TaskStateNew,- swarm.TaskStateAllocated,- swarm.TaskStatePending,- swarm.TaskStateAssigned,- swarm.TaskStateAccepted,- swarm.TaskStatePreparing,- swarm.TaskStateReady,- swarm.TaskStateStarting,- swarm.TaskStateRunning:- return false- }-- return false-}--func (s *Service) upsertStackSourceInternal(ctx context.Context, environmentID, stackName, composeContent, overrideContent, envContent string, files []swarmtypes.SyncFile) error {+func (s *Service) upsertStackSource(ctx context.Context, environmentID, stackName, composeContent, overrideContent, envContent string, files []swarmtypes.SyncFile) error {stackName = strings.TrimSpace(stackName)if stackName == "" {return errors.New("stack name is required")}- rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)+ rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)if err != nil {return err}@@ -742,12 +726,12 @@ func (s *Service) upsertStackSourceInternal(ctx context.Context, environmentID,return nil}-func (s *Service) deleteStackSourceInternal(ctx context.Context, environmentID, stackName string) error {+func (s *Service) deleteStackSource(ctx context.Context, environmentID, stackName string) error {if strings.TrimSpace(stackName) == "" {return errors.New("stack name is required")}- rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)+ rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)if err != nil {return err}@@ -771,18 +755,17 @@ func (s *Service) deleteStackSourceInternal(ctx context.Context, environmentID,return nil}-func normalizeSwarmEnvironmentIDInternal(environmentID string) string {- envID := strings.TrimSpace(environmentID)- return kit.Ternary(envID == "", "0", envID)+func normalizeSwarmEnvironmentID(environmentID string) string {+ return cmp.Or(strings.TrimSpace(environmentID), "0")}-func (s *Service) resolveSwarmStackSourceDirInternal(ctx context.Context, environmentID, stackName string) (string, string, error) {+func (s *Service) resolveSwarmStackSourceDir(ctx context.Context, environmentID, stackName string) (string, string, error) {normalizedStackName := projects.SanitizeProjectName(strings.TrimSpace(stackName))if normalizedStackName == "" || strings.Trim(normalizedStackName, "_") == "" {return "", "", errors.New("invalid stack name")}- rootDir, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDirInternal(ctx, environmentID)+ rootDir, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDir(ctx, environmentID)if err != nil {return "", "", err}@@ -795,8 +778,8 @@ func (s *Service) resolveSwarmStackSourceDirInternal(ctx context.Context, enviroreturn rootDir, stackSourceDir, nil}-func (s *Service) resolveSwarmStackSourceEnvironmentDirInternal(ctx context.Context, environmentID string) (string, string, error) {- normalizedEnvironmentID := projects.SanitizeProjectName(normalizeSwarmEnvironmentIDInternal(environmentID))+func (s *Service) resolveSwarmStackSourceEnvironmentDir(ctx context.Context, environmentID string) (string, string, error) {+ normalizedEnvironmentID := projects.SanitizeProjectName(normalizeSwarmEnvironmentID(environmentID))if normalizedEnvironmentID == "" || strings.Trim(normalizedEnvironmentID, "_") == "" {normalizedEnvironmentID = "0"}@@ -815,22 +798,7 @@ func (s *Service) resolveSwarmStackSourceEnvironmentDirInternal(ctx context.Contreturn rootDir, environmentDir, nil}-func (s *Service) buildStackPaginationConfigInternal() pagination.Config[swarmtypes.StackSummary] {- return pagination.Config[swarmtypes.StackSummary]{- SearchAccessors: []pagination.SearchAccessor[swarmtypes.StackSummary]{- func(stack swarmtypes.StackSummary) (string, error) { return stack.Name, nil },- func(stack swarmtypes.StackSummary) (string, error) { return stack.Namespace, nil },- },- SortBindings: []pagination.SortBinding[swarmtypes.StackSummary]{- {Key: "name", Fn: func(a, b swarmtypes.StackSummary) int { return strings.Compare(a.Name, b.Name) }},- {Key: "services", Fn: func(a, b swarmtypes.StackSummary) int { return cmp.Compare(a.Services, b.Services) }},- {Key: "created", Fn: func(a, b swarmtypes.StackSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},- {Key: "updated", Fn: func(a, b swarmtypes.StackSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},- },- }-}--func classifySwarmStackErrorInternal(err error) error {+func classifySwarmStackError(err error) error {if errors.Is(err, libswarm.ErrInvalidStack) {return common.Classify(common.ErrBadRequest, err)}diff --git a/backend/internal/swarm/children/stacks/service_test.go b/backend/internal/swarm/children/stacks/service_test.goindex 3560cc1bf..f3832924f 100644--- a/backend/internal/swarm/children/stacks/service_test.go+++ b/backend/internal/swarm/children/stacks/service_test.go@@ -16,7 +16,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings")-func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {+func setupSwarmServiceTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -24,7 +24,7 @@ func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForSwarmTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()svc, err := settings.NewSettingsService(ctx, db)if err == nil {@@ -38,12 +38,12 @@ type stackSourceDeployRecorder struct {lastReq swarm.StackDeployRequest}-func stubStackSourceUpdateDeployInternal(t *testing.T) *stackSourceDeployRecorder {+func stubStackSourceUpdateDeploy(t *testing.T) *stackSourceDeployRecorder {t.Helper()- original := deployStackAfterSourceUpdateInternal- t.Cleanup(func() { deployStackAfterSourceUpdateInternal = original })+ original := deployStackAfterSourceUpdate+ t.Cleanup(func() { deployStackAfterSourceUpdate = original })rec := &stackSourceDeployRecorder{}- deployStackAfterSourceUpdateInternal = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {+ deployStackAfterSourceUpdate = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {rec.calls++rec.lastReq = reqreturn &swarm.StackDeployResponse{Name: req.Name}, nil@@ -53,15 +53,15 @@ func stubStackSourceUpdateDeployInternal(t *testing.T) *stackSourceDeployRecordefunc TestService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManager(t *testing.T) {ctx := t.Context()- db := setupSwarmServiceTestDBInternal(t)+ db := setupSwarmServiceTestDB(t)rootDir := t.TempDir()t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)require.NoError(t, err)svc := NewService(nil, nil, nil, nil, nil, settingsSvc)- deployRec := stubStackSourceUpdateDeployInternal(t)+ deployRec := stubStackSourceUpdateDeploy(t)updated, err := svc.UpdateStackSource(ctx, "0", "demo-stack", swarm.StackSourceUpdateRequest{ComposeContent: "services:\n web:\n image: nginx:alpine\n",@@ -117,15 +117,15 @@ func TestService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManager(t *tfunc TestService_UpdateAndGetStackSource_RoundTripsOverride(t *testing.T) {ctx := t.Context()- db := setupSwarmServiceTestDBInternal(t)+ db := setupSwarmServiceTestDB(t)rootDir := t.TempDir()t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)require.NoError(t, err)svc := NewService(nil, nil, nil, nil, nil, settingsSvc)- stubStackSourceUpdateDeployInternal(t)+ stubStackSourceUpdateDeploy(t)overridePath := filepath.Join(rootDir, "0", "demo-stack", "compose.override.yaml")@@ -157,20 +157,20 @@ func TestService_UpdateAndGetStackSource_RoundTripsOverride(t *testing.T) {func TestService_UpdateStackSource_PrunesAndRestoresOnDeployFailure(t *testing.T) {ctx := t.Context()- db := setupSwarmServiceTestDBInternal(t)+ db := setupSwarmServiceTestDB(t)rootDir := t.TempDir()t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)require.NoError(t, err)svc := NewService(nil, nil, nil, nil, nil, settingsSvc)- original := deployStackAfterSourceUpdateInternal- t.Cleanup(func() { deployStackAfterSourceUpdateInternal = original })+ original := deployStackAfterSourceUpdate+ t.Cleanup(func() { deployStackAfterSourceUpdate = original })var lastReq swarm.StackDeployRequestvar deployErr error- deployStackAfterSourceUpdateInternal = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {+ deployStackAfterSourceUpdate = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {lastReq = reqif deployErr != nil {return nil, deployErrdiff --git a/backend/internal/swarm/service.go b/backend/internal/swarm/service.goindex eda963a8c..6c3b5346f 100644--- a/backend/internal/swarm/service.go+++ b/backend/internal/swarm/service.go@@ -2,6 +2,7 @@ package swarmimport ("bytes"+ "cmp""context"stdjson "encoding/json""encoding/json/v2"@@ -15,7 +16,6 @@ import ("github.com/moby/moby/api/types/swarm""github.com/moby/moby/api/types/system""github.com/moby/moby/client"- "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/docker"@@ -63,11 +63,11 @@ func NewSwarmService(registryAuth = registryService.GetRegistryAuthForImage}- s.services = services.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal, s.listTasksPaginatedWithFiltersInternal)- s.nodes = nodes.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal, s.listTasksPaginatedWithFiltersInternal, s.GetSwarmJoinTokens, environmentService)- s.stacks = stacks.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal, s.listTasksPaginatedWithFiltersInternal, s.services.PaginateSummaries, registryAuth, settingsService)- s.configs = configs.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal)- s.secrets = secrets.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal)+ s.services = services.NewService(dockerService, s.ensureSwarmManager, s.listTasksPaginatedWithFilters)+ s.nodes = nodes.NewService(dockerService, s.ensureSwarmManager, s.listTasksPaginatedWithFilters, s.GetSwarmJoinTokens, environmentService)+ s.stacks = stacks.NewService(dockerService, s.ensureSwarmManager, s.listTasksPaginatedWithFilters, s.services.PaginateSummaries, registryAuth, settingsService)+ s.configs = configs.NewService(dockerService, s.ensureSwarmManager)+ s.secrets = secrets.NewService(dockerService, s.ensureSwarmManager)return s}@@ -95,24 +95,21 @@ func (s *SwarmService) IsEnabled(ctx context.Context) (bool, error) {}func (s *SwarmService) ListTasksPaginated(ctx context.Context, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {- if err := s.ensureSwarmManagerInternal(ctx); err != nil {+ if err := s.ensureSwarmManager(ctx); err != nil {return nil, pagination.Response{}, err}- return s.listTasksPaginatedWithFiltersInternal(ctx, nil, params)+ return s.listTasksPaginatedWithFilters(ctx, nil, params)}func (s *SwarmService) GetSwarmInfo(ctx context.Context) (*swarmtypes.SwarmInfo, error) {- if err := s.ensureSwarmManagerInternal(ctx); err != nil {+ if err := s.ensureSwarmManager(ctx); err != nil {return nil, err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- infoResult, err := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm: %w", err)}@@ -121,12 +118,7 @@ func (s *SwarmService) GetSwarmInfo(ctx context.Context) (*swarmtypes.SwarmInfo,}func (s *SwarmService) InitSwarm(ctx context.Context, req swarmtypes.SwarmInitRequest) (*swarmtypes.SwarmInitResponse, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- spec, err := decodeSwarmSpecInternal(req.Spec)+ spec, err := decodeSwarmSpec(req.Spec)if err != nil {return nil, err}@@ -140,79 +132,72 @@ func (s *SwarmService) InitSwarm(ctx context.Context, req swarmtypes.SwarmInitRedefaultAddrPool = append(defaultAddrPool, prefix.Masked())}- initResult, err := dockerClient.SwarmInit(ctx, client.SwarmInitOptions{- ListenAddr: defaultSwarmListenAddrInternal(req.ListenAddr),- AdvertiseAddr: req.AdvertiseAddr,- DataPathAddr: req.DataPathAddr,- DataPathPort: req.DataPathPort,- ForceNewCluster: req.ForceNewCluster,- Spec: spec,- AutoLockManagers: req.AutoLockManagers,- Availability: req.Availability,- DefaultAddrPool: defaultAddrPool,- SubnetSize: req.SubnetSize,+ initResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInitResult, error) {+ return dockerClient.SwarmInit(ctx, client.SwarmInitOptions{+ ListenAddr: cmp.Or(strings.TrimSpace(req.ListenAddr), defaultSwarmListenAddr),+ AdvertiseAddr: req.AdvertiseAddr,+ DataPathAddr: req.DataPathAddr,+ DataPathPort: req.DataPathPort,+ ForceNewCluster: req.ForceNewCluster,+ Spec: spec,+ AutoLockManagers: req.AutoLockManagers,+ Availability: req.Availability,+ DefaultAddrPool: defaultAddrPool,+ SubnetSize: req.SubnetSize,+ })})if err != nil {return nil, fmt.Errorf("failed to initialize swarm: %w", err)}- s.persistSwarmEnabledStateInternal(ctx, true)+ s.persistSwarmEnabledState(ctx, true)return &swarmtypes.SwarmInitResponse{NodeID: initResult.NodeID}, nil}func (s *SwarmService) JoinSwarm(ctx context.Context, req swarmtypes.SwarmJoinRequest) error {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, swarmJoinErr := dockerClient.SwarmJoin(ctx, client.SwarmJoinOptions{- ListenAddr: defaultSwarmListenAddrInternal(req.ListenAddr),- AdvertiseAddr: req.AdvertiseAddr,- DataPathAddr: req.DataPathAddr,- RemoteAddrs: req.RemoteAddrs,- JoinToken: req.JoinToken,- Availability: req.Availability,+ if _, swarmJoinErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmJoinResult, error) {+ return dockerClient.SwarmJoin(ctx, client.SwarmJoinOptions{+ ListenAddr: cmp.Or(strings.TrimSpace(req.ListenAddr), defaultSwarmListenAddr),+ AdvertiseAddr: req.AdvertiseAddr,+ DataPathAddr: req.DataPathAddr,+ RemoteAddrs: req.RemoteAddrs,+ JoinToken: req.JoinToken,+ Availability: req.Availability,+ })}); swarmJoinErr != nil {return fmt.Errorf("failed to join swarm: %w", swarmJoinErr)}- s.persistSwarmEnabledStateInternal(ctx, true)+ s.persistSwarmEnabledState(ctx, true)return nil}func (s *SwarmService) LeaveSwarm(ctx context.Context, req swarmtypes.SwarmLeaveRequest) error {- if err := s.ensureSwarmActiveInternal(ctx); err != nil {+ if _, err := s.ensureSwarmActive(ctx); err != nil {return err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, swarmLeaveErr := dockerClient.SwarmLeave(ctx, client.SwarmLeaveOptions{Force: req.Force}); swarmLeaveErr != nil {+ if _, swarmLeaveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmLeaveResult, error) {+ return dockerClient.SwarmLeave(ctx, client.SwarmLeaveOptions{Force: req.Force})+ }); swarmLeaveErr != nil {return fmt.Errorf("failed to leave swarm: %w", swarmLeaveErr)}- s.persistSwarmEnabledStateInternal(ctx, false)+ s.persistSwarmEnabledState(ctx, false)return nil}func (s *SwarmService) UnlockSwarm(ctx context.Context, req swarmtypes.SwarmUnlockRequest) error {- if err := s.ensureSwarmActiveInternal(ctx); err != nil {+ if _, err := s.ensureSwarmActive(ctx); err != nil {return err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- if _, swarmUnlockErr := dockerClient.SwarmUnlock(ctx, client.SwarmUnlockOptions{Key: req.Key}); swarmUnlockErr != nil {+ if _, swarmUnlockErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmUnlockResult, error) {+ return dockerClient.SwarmUnlock(ctx, client.SwarmUnlockOptions{Key: req.Key})+ }); swarmUnlockErr != nil {return fmt.Errorf("failed to unlock swarm: %w", swarmUnlockErr)}@@ -220,16 +205,13 @@ func (s *SwarmService) UnlockSwarm(ctx context.Context, req swarmtypes.SwarmUnlo}func (s *SwarmService) GetSwarmUnlockKey(ctx context.Context) (*swarmtypes.SwarmUnlockKeyResponse, error) {- if err := s.ensureSwarmManagerInternal(ctx); err != nil {+ if err := s.ensureSwarmManager(ctx); err != nil {return nil, err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- unlockResult, err := dockerClient.SwarmGetUnlockKey(ctx)+ unlockResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmGetUnlockKeyResult, error) {+ return dockerClient.SwarmGetUnlockKey(ctx)+ })if err != nil {return nil, fmt.Errorf("failed to get swarm unlock key: %w", err)}@@ -238,16 +220,13 @@ func (s *SwarmService) GetSwarmUnlockKey(ctx context.Context) (*swarmtypes.Swarm}func (s *SwarmService) GetSwarmJoinTokens(ctx context.Context) (*swarmtypes.SwarmJoinTokensResponse, error) {- if err := s.ensureSwarmManagerInternal(ctx); err != nil {+ if err := s.ensureSwarmManager(ctx); err != nil {return nil, err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- infoResult, err := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to inspect swarm: %w", err)}@@ -259,16 +238,13 @@ func (s *SwarmService) GetSwarmJoinTokens(ctx context.Context) (*swarmtypes.Swar}func (s *SwarmService) RotateSwarmJoinTokens(ctx context.Context, req swarmtypes.SwarmRotateJoinTokensRequest) error {- if err := s.ensureSwarmManagerInternal(ctx); err != nil {+ if err := s.ensureSwarmManager(ctx); err != nil {return err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-- infoResult, err := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ })if err != nil {return fmt.Errorf("failed to inspect swarm: %w", err)}@@ -280,11 +256,13 @@ func (s *SwarmService) RotateSwarmJoinTokens(ctx context.Context, req swarmtypesrotateManager = true}- if _, swarmUpdateErr := dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{- Version: infoResult.Swarm.Version,- Spec: infoResult.Swarm.Spec,- RotateWorkerToken: rotateWorker,- RotateManagerToken: rotateManager,+ if _, swarmUpdateErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmUpdateResult, error) {+ return dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{+ Version: infoResult.Swarm.Version,+ Spec: infoResult.Swarm.Spec,+ RotateWorkerToken: rotateWorker,+ RotateManagerToken: rotateManager,+ })}); swarmUpdateErr != nil {return fmt.Errorf("failed to rotate swarm join tokens: %w", swarmUpdateErr)}@@ -293,35 +271,34 @@ func (s *SwarmService) RotateSwarmJoinTokens(ctx context.Context, req swarmtypes}func (s *SwarmService) UpdateSwarmSpec(ctx context.Context, req swarmtypes.SwarmUpdateRequest) error {- if err := s.ensureSwarmManagerInternal(ctx); err != nil {+ if err := s.ensureSwarmManager(ctx); err != nil {return err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }-version := req.Versionif version == 0 {- infoResult, swarmInspectErr := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ infoResult, swarmInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})+ })if swarmInspectErr != nil {return fmt.Errorf("failed to inspect swarm: %w", swarmInspectErr)}version = infoResult.Swarm.Version.Index}- spec, err := decodeSwarmSpecInternal(req.Spec)+ spec, err := decodeSwarmSpec(req.Spec)if err != nil {return err}- if _, swarmUpdateErr := dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{- Version: swarm.Version{Index: version},- Spec: spec,- RotateWorkerToken: req.RotateWorkerToken,- RotateManagerToken: req.RotateManagerToken,- RotateManagerUnlockKey: req.RotateManagerUnlockKey,+ if _, swarmUpdateErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmUpdateResult, error) {+ return dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{+ Version: swarm.Version{Index: version},+ Spec: spec,+ RotateWorkerToken: req.RotateWorkerToken,+ RotateManagerToken: req.RotateManagerToken,+ RotateManagerUnlockKey: req.RotateManagerUnlockKey,+ })}); swarmUpdateErr != nil {return fmt.Errorf("failed to update swarm spec: %w", swarmUpdateErr)}@@ -329,13 +306,10 @@ func (s *SwarmService) UpdateSwarmSpec(ctx context.Context, req swarmtypes.Swarmreturn nil}-func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{})+func (s *SwarmService) listTasksPaginatedWithFilters(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {+ servicesResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{})+ })if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list swarm services: %w", err)}@@ -345,7 +319,9 @@ func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.ContextserviceNameByID[service.ID] = service.Spec.Name}- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})+ nodesResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {+ return dockerClient.NodeList(ctx, client.NodeListOptions{})+ })if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list swarm nodes: %w", err)}@@ -358,7 +334,9 @@ func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Contextif filters == nil {filters = make(client.Filters)}- tasksResult, err := dockerClient.TaskList(ctx, client.TaskListOptions{Filters: filters})+ tasksResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {+ return dockerClient.TaskList(ctx, client.TaskListOptions{Filters: filters})+ })if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list swarm tasks: %w", err)}@@ -368,13 +346,28 @@ func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Contextitems = append(items, swarmtypes.NewTaskSummary(task, serviceNameByID[task.ServiceID], nodeNameByID[task.NodeID]))}- config := s.buildTaskPaginationConfigInternal()+ config := pagination.Config[swarmtypes.TaskSummary]{+ SearchAccessors: []pagination.SearchAccessor[swarmtypes.TaskSummary]{+ func(task swarmtypes.TaskSummary) (string, error) { return task.Name, nil },+ func(task swarmtypes.TaskSummary) (string, error) { return task.ServiceName, nil },+ func(task swarmtypes.TaskSummary) (string, error) { return task.NodeName, nil },+ func(task swarmtypes.TaskSummary) (string, error) { return task.ID, nil },+ func(task swarmtypes.TaskSummary) (string, error) { return task.CurrentState, nil },+ },+ SortBindings: []pagination.SortBinding[swarmtypes.TaskSummary]{+ {Key: "service", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.ServiceName, b.ServiceName) }},+ {Key: "node", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.NodeName, b.NodeName) }},+ {Key: "state", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.CurrentState, b.CurrentState) }},+ {Key: "created", Fn: func(a, b swarmtypes.TaskSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},+ {Key: "updated", Fn: func(a, b swarmtypes.TaskSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},+ },+ }result := config.SearchOrderAndPaginate(items, params)paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params)return result.Items, paginationResp, nil}-func decodeSwarmSpecInternal(raw stdjson.RawMessage) (swarm.Spec, error) {+func decodeSwarmSpec(raw stdjson.RawMessage) (swarm.Spec, error) {trimmed := bytes.TrimSpace(raw)if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {return swarm.Spec{}, errors.New("swarm spec is required")@@ -392,20 +385,11 @@ func decodeSwarmSpecInternal(raw stdjson.RawMessage) (swarm.Spec, error) {return spec, nil}-func defaultSwarmListenAddrInternal(listenAddr string) string {- trimmed := strings.TrimSpace(listenAddr)- return kit.Ternary(trimmed == "", defaultSwarmListenAddr, trimmed)-}--func (s *SwarmService) ensureSwarmManagerInternal(ctx context.Context) error {- info, err := s.getDockerInfoInternal(ctx)+func (s *SwarmService) ensureSwarmManager(ctx context.Context) error {+ info, err := s.ensureSwarmActive(ctx)if err != nil {return err}-- if info.Swarm.LocalNodeState != swarm.LocalNodeStateActive {- return common.Classify(common.ErrSwarmNotEnabled, errors.New("Swarm mode is not enabled")) //nolint:staticcheck // Preserve the existing error message.- }if !info.Swarm.ControlAvailable {return common.Classify(common.ErrSwarmManagerRequired, errors.New("Swarm manager access required")) //nolint:staticcheck // Preserve the existing error message.}@@ -413,26 +397,21 @@ func (s *SwarmService) ensureSwarmManagerInternal(ctx context.Context) error {return nil}-func (s *SwarmService) ensureSwarmActiveInternal(ctx context.Context) error {- info, err := s.getDockerInfoInternal(ctx)+func (s *SwarmService) ensureSwarmActive(ctx context.Context) (system.Info, error) {+ info, err := s.getDockerInfo(ctx)if err != nil {- return err+ return system.Info{}, err}-if info.Swarm.LocalNodeState != swarm.LocalNodeStateActive {- return common.Classify(common.ErrSwarmNotEnabled, errors.New("Swarm mode is not enabled")) //nolint:staticcheck // Preserve the existing error message.+ return system.Info{}, common.Classify(common.ErrSwarmNotEnabled, errors.New("Swarm mode is not enabled")) //nolint:staticcheck // Preserve the existing error message.}-- return nil+ return info, nil}-func (s *SwarmService) getDockerInfoInternal(ctx context.Context) (system.Info, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return system.Info{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})+func (s *SwarmService) getDockerInfo(ctx context.Context) (system.Info, error) {+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SystemInfoResult, error) {+ return dockerClient.Info(ctx, client.InfoOptions{})+ })if err != nil {return system.Info{}, fmt.Errorf("failed to get Docker info: %w", err)}@@ -441,7 +420,7 @@ func (s *SwarmService) getDockerInfoInternal(ctx context.Context) (system.Info,}func (s *SwarmService) SyncSwarmEnabledState(ctx context.Context) error {- info, err := s.getDockerInfoInternal(ctx)+ info, err := s.getDockerInfo(ctx)if err != nil {return err}@@ -458,7 +437,7 @@ func (s *SwarmService) SyncSwarmEnabledState(ctx context.Context) error {return nil}-func (s *SwarmService) persistSwarmEnabledStateInternal(ctx context.Context, enabled bool) {+func (s *SwarmService) persistSwarmEnabledState(ctx context.Context, enabled bool) {if s.kvService == nil {return}@@ -467,22 +446,3 @@ func (s *SwarmService) persistSwarmEnabledStateInternal(ctx context.Context, enaslog.WarnContext(ctx, "Failed to persist swarm enabled state", "enabled", enabled, "error", err)}}--func (s *SwarmService) buildTaskPaginationConfigInternal() pagination.Config[swarmtypes.TaskSummary] {- return pagination.Config[swarmtypes.TaskSummary]{- SearchAccessors: []pagination.SearchAccessor[swarmtypes.TaskSummary]{- func(task swarmtypes.TaskSummary) (string, error) { return task.Name, nil },- func(task swarmtypes.TaskSummary) (string, error) { return task.ServiceName, nil },- func(task swarmtypes.TaskSummary) (string, error) { return task.NodeName, nil },- func(task swarmtypes.TaskSummary) (string, error) { return task.ID, nil },- func(task swarmtypes.TaskSummary) (string, error) { return task.CurrentState, nil },- },- SortBindings: []pagination.SortBinding[swarmtypes.TaskSummary]{- {Key: "service", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.ServiceName, b.ServiceName) }},- {Key: "node", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.NodeName, b.NodeName) }},- {Key: "state", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.CurrentState, b.CurrentState) }},- {Key: "created", Fn: func(a, b swarmtypes.TaskSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},- {Key: "updated", Fn: func(a, b swarmtypes.TaskSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},- },- }-}diff --git a/backend/internal/swarm/service_test.go b/backend/internal/swarm/service_test.goindex 4af3cf479..7f1365cf5 100644--- a/backend/internal/swarm/service_test.go+++ b/backend/internal/swarm/service_test.go@@ -7,20 +7,14 @@ import ("github.com/stretchr/testify/require")-func TestDecodeSwarmSpecInternal_AllowsEmptyObject(t *testing.T) {- spec, err := decodeSwarmSpecInternal(json.RawMessage(`{}`))+func TestDecodeSwarmSpec_AllowsEmptyObject(t *testing.T) {+ spec, err := decodeSwarmSpec(json.RawMessage(`{}`))require.NoError(t, err)require.NotNil(t, spec.Labels)require.Empty(t, spec.Labels)}-func TestDecodeSwarmSpecInternal_RejectsNull(t *testing.T) {- _, err := decodeSwarmSpecInternal(json.RawMessage(`null`))+func TestDecodeSwarmSpec_RejectsNull(t *testing.T) {+ _, err := decodeSwarmSpec(json.RawMessage(`null`))require.EqualError(t, err, "swarm spec is required")}--func TestDefaultSwarmListenAddrInternal(t *testing.T) {- require.Equal(t, defaultSwarmListenAddr, defaultSwarmListenAddrInternal(""))- require.Equal(t, defaultSwarmListenAddr, defaultSwarmListenAddrInternal(" "))- require.Equal(t, "eth0:2377", defaultSwarmListenAddrInternal(" eth0:2377 "))-}diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.goindex cb3522171..1403bdd09 100644--- a/backend/internal/system/children/backup/service.go+++ b/backend/internal/system/children/backup/service.go@@ -1041,7 +1041,9 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useif err != nil {return errors.New("arcane system restore requires Arcane to run in Docker")}- inspectResult, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ inspectResult, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerID, client.ContainerInspectOptions{})+ })if err != nil {return fmt.Errorf("inspect Arcane container: %w", err)}@@ -1153,28 +1155,34 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, useClusterOptions: executableMount.ClusterOptions,})}- created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: &container.Config{- Image: current.Config.Image,- Cmd: []string{kit.Ternary(executableMount != nil, systemRecoveryHelperPath, executablePath), "recovery-restore", "--request", requestFile},- User: "0:0",- Env: containerEnv,- Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},- },- HostConfig: &container.HostConfig{- AutoRemove: true,- Mounts: mounts,- NetworkMode: networkMode,- SecurityOpt: slices.Clone(kit.FromPtr(current.HostConfig).SecurityOpt),- Privileged: kit.FromPtr(current.HostConfig).Privileged,- },- Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),+ created, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &container.Config{+ Image: current.Config.Image,+ Cmd: []string{kit.Ternary(executableMount != nil, systemRecoveryHelperPath, executablePath), "recovery-restore", "--request", requestFile},+ User: "0:0",+ Env: containerEnv,+ Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},+ },+ HostConfig: &container.HostConfig{+ AutoRemove: true,+ Mounts: mounts,+ NetworkMode: networkMode,+ SecurityOpt: slices.Clone(kit.FromPtr(current.HostConfig).SecurityOpt),+ Privileged: kit.FromPtr(current.HostConfig).Privileged,+ },+ Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),+ })})if err != nil {return fmt.Errorf("create recovery helper container: %w", err)}- if _, containerStartErr := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); containerStartErr != nil {- _, _ = dockerClient.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {+ return cli.ContainerStart(ctx, created.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {+ return cli.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})+ })return fmt.Errorf("start recovery helper container: %w", containerStartErr)}cleanupRequest = falsediff --git a/backend/internal/system/children/prune/service.go b/backend/internal/system/children/prune/service.goindex 2b0ea382b..adbc25456 100644--- a/backend/internal/system/children/prune/service.go+++ b/backend/internal/system/children/prune/service.go@@ -26,6 +26,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate""github.com/getarcaneapp/arcane/backend/v2/internal/network""github.com/getarcaneapp/arcane/backend/v2/internal/volume"@@ -38,7 +39,7 @@ import (// Service runs manual and scheduled prunes, serializing them per environment.type Service struct {location *time.Location- dockerClient func(ctx context.Context) (*client.Client, error)+ dockerService *docker.DockerClientServiceactivityService *activity.ActivityServiceimageUpdateService *imageupdate.ImageUpdateServicevolumeService *volume.VolumeService@@ -49,7 +50,7 @@ type Service struct {func NewService(location *time.Location,- dockerClient func(ctx context.Context) (*client.Client, error),+ dockerService *docker.DockerClientService,activityService *activity.ActivityService,imageUpdateService *imageupdate.ImageUpdateService,volumeService *volume.VolumeService,@@ -57,7 +58,7 @@ func NewService() *Service {return &Service{location: location,- dockerClient: dockerClient,+ dockerService: dockerService,activityService: activityService,imageUpdateService: imageUpdateService,volumeService: volumeService,@@ -340,11 +341,6 @@ func (s *Service) completeSystemPruneActivity(ctx context.Context, activityID st}func (s *Service) pruneContainers(ctx context.Context, options system.PruneContainersOptions) (system.PruneAllResult, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-filterArgs := make(client.Filters)if options.Mode == system.PruneContainerModeOlderThan {if strings.TrimSpace(options.Until) == "" {@@ -353,7 +349,11 @@ func (s *Service) pruneContainers(ctx context.Context, options system.PruneContafilterArgs = filterArgs.Add("until", options.Until)}- report, err := dockerClient.ContainerPrune(ctx, client.ContainerPruneOptions{Filters: filterArgs})+ cli, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return system.PruneAllResult{}, fmt.Errorf("failed to prune containers: %w", err)+ }+ report, err := cli.ContainerPrune(ctx, client.ContainerPruneOptions{Filters: filterArgs})if err != nil {return system.PruneAllResult{}, fmt.Errorf("failed to prune containers: %w", err)}@@ -366,11 +366,6 @@ func (s *Service) pruneContainers(ctx context.Context, options system.PruneConta}func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOptions) (system.PruneAllResult, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-filterArgs := make(client.Filters)switch options.Mode {case system.PruneImageModeNone:@@ -389,7 +384,11 @@ func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOptreturn system.PruneAllResult{}, fmt.Errorf("unsupported image prune mode: %s", options.Mode)}- report, err := dockerClient.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})+ cli, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return system.PruneAllResult{}, fmt.Errorf("failed to prune images: %w", err)+ }+ report, err := cli.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})if err != nil {return system.PruneAllResult{}, fmt.Errorf("failed to prune images: %w", err)}@@ -418,17 +417,17 @@ func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOpt}func (s *Service) pruneBuildCache(ctx context.Context, options system.PruneBuildCacheOptions) (system.PruneAllResult, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- slog.ErrorContext(ctx, "Error connecting to Docker for build cache prune", "error", err.Error())- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker for build cache prune: %w", err)- }if options.Mode == system.PruneBuildCacheModeOlderThan && strings.TrimSpace(options.Until) == "" {return system.PruneAllResult{}, errors.New("build cache prune mode olderThan requires until")}slog.DebugContext(ctx, "starting build cache pruning", "mode", options.Mode, "until", options.Until)- report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{+ cli, err := s.dockerService.GetClient(ctx)+ if err != nil {+ slog.ErrorContext(ctx, "Error pruning build cache", "error", err.Error())+ return system.PruneAllResult{}, fmt.Errorf("failed to prune build cache: %w", err)+ }+ report, err := cli.BuildCachePrune(ctx, client.BuildCachePruneOptions{All: options.Mode == system.PruneBuildCacheModeAll,Filters: kit.Ternary(options.Mode == system.PruneBuildCacheModeOlderThan, make(client.Filters).Add("until", options.Until), nil),})@@ -461,11 +460,6 @@ func (s *Service) pruneVolumes(ctx context.Context, options system.PruneVolumesO}func (s *Service) pruneNetworks(ctx context.Context, options system.PruneNetworksOptions) (system.PruneAllResult, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)- }-filterArgs := make(client.Filters)if options.Mode == system.PruneNetworkModeOlderThan {if strings.TrimSpace(options.Until) == "" {@@ -474,7 +468,11 @@ func (s *Service) pruneNetworks(ctx context.Context, options system.PruneNetworkfilterArgs = filterArgs.Add("until", options.Until)}- report, err := dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: filterArgs})+ cli, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return system.PruneAllResult{}, fmt.Errorf("failed to prune networks: %w", err)+ }+ report, err := cli.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: filterArgs})if err != nil {return system.PruneAllResult{}, fmt.Errorf("failed to prune networks: %w", err)}@@ -546,19 +544,17 @@ func (s *Service) PruneScheduled(ctx context.Context, environmentID string, req}func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequest) (prunePlan, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return prunePlan{}, err- }if req.Volumes != nil {s.volumeService.CleanupHelperContainers(ctx)}- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{- Containers: true,- Images: true,- Volumes: true,- BuildCache: req.BuildCache != nil,- Verbose: true,+ usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {+ return cli.DiskUsage(ctx, client.DiskUsageOptions{+ Containers: true,+ Images: true,+ Volumes: true,+ BuildCache: req.BuildCache != nil,+ Verbose: true,+ })})if err != nil {return prunePlan{}, err@@ -572,7 +568,7 @@ func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequeif err != nil {return prunePlan{}, err}- networks, err := s.selectPruneNetworks(ctx, dockerClient, req, selectedContainers)+ networks, err := s.selectPruneNetworks(ctx, req, selectedContainers)if err != nil {return prunePlan{}, err}@@ -656,7 +652,7 @@ func (s *Service) selectPruneVolumes(req system.PruneAllRequest, usage client.Direturn resources}-func (s *Service) selectPruneNetworks(ctx context.Context, dockerClient *client.Client, req system.PruneAllRequest, selectedContainers map[string]bool) ([]pruneResource, error) {+func (s *Service) selectPruneNetworks(ctx context.Context, req system.PruneAllRequest, selectedContainers map[string]bool) ([]pruneResource, error) {if req.Networks == nil || req.Networks.Mode == system.PruneNetworkModeNone {return nil, nil}@@ -664,7 +660,9 @@ func (s *Service) selectPruneNetworks(ctx context.Context, dockerClient *client.if err != nil {return nil, err}- networks, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})+ networks, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NetworkListResult, error) {+ return cli.NetworkList(ctx, client.NetworkListOptions{})+ })if err != nil {return nil, err}@@ -674,7 +672,9 @@ nextNetwork:if item.Name == "bridge" || item.Name == "host" || item.Name == "none" || item.Ingress || item.Scope != "local" || (!until.IsZero() && !item.Created.Before(until)) {continue}- inspect, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, item.ID, client.NetworkInspectOptions{})+ inspect, networkInspectWithCompatibilityErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NetworkInspectResult, error) {+ return compat.NetworkInspectWithCompatibility(ctx, cli, item.ID, client.NetworkInspectOptions{})+ })if networkInspectWithCompatibilityErr != nil {return nil, networkInspectWithCompatibilityErr}@@ -778,10 +778,6 @@ func (s *Service) executePrunePlan(ActivityID: mo.EmptyableToOption(activityID).ToPointer(),}, err}- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return finish(err)- }for _, resource := range plan.Resources {if resource.Kind == "build_cache" {continue@@ -799,7 +795,7 @@ func (s *Service) executePrunePlan(}); progressErr != nil {return finish(progressErr)}- size, eligible, exists, eligibleErr := s.pruneResourceEligible(ctx, dockerClient, plan, resource)+ size, eligible, exists, eligibleErr := s.pruneResourceEligible(ctx, plan, resource)if eligibleErr != nil {return finish(eligibleErr)}@@ -807,7 +803,7 @@ func (s *Service) executePrunePlan(if !exists || !eligible {return nil}- return s.removePruneResource(ctx, dockerClient, resource)+ return s.removePruneResource(ctx, resource)}()removeFailed := removeErr != nil && !errdefs.IsNotFound(removeErr)if removeFailed {@@ -856,20 +852,24 @@ func (s *Service) executePrunePlan(return finish(deleteRecordsForImagesErr)}}- reclaimed, err := s.executePruneCache(ctx, dockerClient, plan, recovery)+ reclaimed, err := s.executePruneCache(ctx, plan, recovery)spaceReclaimed += reclaimedbuildCacheSpace += reclaimedreturn finish(err)}-func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.Client, resource pruneResource) error {+func (s *Service) removePruneResource(ctx context.Context, resource pruneResource) error {switch resource.Kind {case "container":- _, err := dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {+ return cli.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})+ })return errcase "image":for _, name := range resource.Names {- inspected, inspectErr := dockerClient.ImageInspect(ctx, name)+ inspected, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {+ return cli.ImageInspect(ctx, name)+ })if errdefs.IsNotFound(inspectErr) {continue}@@ -879,11 +879,15 @@ func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.if inspected.ID != resource.ID {continue}- if _, err := dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false}); err != nil {+ if _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageRemoveResult, error) {+ return cli.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false})+ }); err != nil {return err}}- _, err := dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageRemoveResult, error) {+ return cli.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})+ })return errcase "volume":return s.volumeService.DeleteVolume(ctx, resource.ID, false, user.SystemUser)@@ -894,13 +898,15 @@ func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.}}-func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Client, plan prunePlan, recovery bool) (uint64, error) {+func (s *Service) executePruneCache(ctx context.Context, plan prunePlan, recovery bool) (uint64, error) {request := plan.Request.BuildCacheif request == nil || request.Mode == system.PruneBuildCacheModeNone {return 0, nil}if recovery {- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})+ usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {+ return cli.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})+ })if err != nil {return 0, err}@@ -927,7 +933,11 @@ func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Clif request.Mode == system.PruneBuildCacheModeOlderThan {filters = filters.Add("until", request.Until)}- report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{+ cli, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return reclaimed, err+ }+ report, err := cli.BuildCachePrune(ctx, client.BuildCachePruneOptions{All: request.Mode == system.PruneBuildCacheModeAll,Filters: filters,})@@ -940,9 +950,11 @@ func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Cl}// pruneResourceEligible re-checks a planned resource against live state and returns its size from that snapshot.-func (s *Service) pruneResourceEligible(ctx context.Context, dockerClient *client.Client, plan prunePlan, resource pruneResource) (size int64, eligible, exists bool, err error) {+func (s *Service) pruneResourceEligible(ctx context.Context, plan prunePlan, resource pruneResource) (size int64, eligible, exists bool, err error) {if resource.Kind == "network" {- inspected, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, resource.ID, client.NetworkInspectOptions{})+ inspected, networkInspectWithCompatibilityErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NetworkInspectResult, error) {+ return compat.NetworkInspectWithCompatibility(ctx, cli, resource.ID, client.NetworkInspectOptions{})+ })if errdefs.IsNotFound(networkInspectWithCompatibilityErr) {return resource.Size, false, false, nil}@@ -951,11 +963,13 @@ func (s *Service) pruneResourceEligible(ctx context.Context, dockerClient *clien}return resource.Size, len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil}- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{- Containers: resource.Kind == "container",- Images: resource.Kind == "image",- Volumes: resource.Kind == "volume",- Verbose: true,+ usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {+ return cli.DiskUsage(ctx, client.DiskUsageOptions{+ Containers: resource.Kind == "container",+ Images: resource.Kind == "image",+ Volumes: resource.Kind == "volume",+ Verbose: true,+ })})if err != nil {return resource.Size, false, false, errdiff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.goindex 1c2a0644b..faf16bbf1 100644--- a/backend/internal/system/children/upgrade/service.go+++ b/backend/internal/system/children/upgrade/service.go@@ -262,12 +262,16 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targ}return *self, nil}- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ inspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerID, client.ContainerInspectOptions{})+ })if inspectErr == nil {return inspect.Container, nil}// Fall back to a prefix match across every container.- containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ containers, listErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerListResult, error) {+ return cli.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if listErr != nil {return container.InspectResponse{}, fmt.Errorf("inspect container: %w", listErr)}@@ -275,7 +279,9 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targif idx < 0 {return container.InspectResponse{}, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))}- if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {+ if inspect, inspectErr = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, cli, containers.Items[idx].ID, client.ContainerInspectOptions{})+ }); inspectErr != nil {return container.InspectResponse{}, fmt.Errorf("inspect container: %w", inspectErr)}return inspect.Container, nil@@ -409,7 +415,9 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgr// On SELinux-enforcing hosts the socket is container_var_run_t, so without a label opt the upgrader exits with EACCES.if !kit.FromPtr(prepared.current.HostConfig).Privileged &&!slices.ContainsFunc(securityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") }) {- infoResult, infoErr := dockerClient.Info(ctx, client.InfoOptions{})+ infoResult, infoErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.SystemInfoResult, error) {+ return cli.Info(ctx, client.InfoOptions{})+ })if infoErr != nil {slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", infoErr)} else if slices.Contains(infoResult.Info.SecurityOptions, "name=selinux") {@@ -418,43 +426,49 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgr}upgraderName := fmt.Sprintf("%s-upgrader-%d", prepared.containerName, time.Now().Unix())- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: &container.Config{- Image: prepared.pullImage,- Cmd: upgradeCmd,- // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.- User: "0:0",- Env: containerEnv,- Labels: map[string]string{- "com.getarcaneapp.arcane.upgrader": "true",- "com.getarcaneapp.arcane": "true",- },- },- HostConfig: &container.HostConfig{- AutoRemove: !keepUpgraderContainer,- Mounts: mounts,- NetworkMode: networkMode,- SecurityOpt: securityOpt,- Privileged: kit.FromPtr(prepared.current.HostConfig).Privileged,- },- NetworkingConfig: func() *network.NetworkingConfig {- if networkMode == "" || !networkMode.IsUserDefined() {- return nil- }- return &network.NetworkingConfig{- EndpointsConfig: map[string]*network.EndpointSettings{- string(networkMode): {Aliases: []string{"arcane-upgrader"}},+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &container.Config{+ Image: prepared.pullImage,+ Cmd: upgradeCmd,+ // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.+ User: "0:0",+ Env: containerEnv,+ Labels: map[string]string{+ "com.getarcaneapp.arcane.upgrader": "true",+ "com.getarcaneapp.arcane": "true",},- }- }(),- Name: upgraderName,+ },+ HostConfig: &container.HostConfig{+ AutoRemove: !keepUpgraderContainer,+ Mounts: mounts,+ NetworkMode: networkMode,+ SecurityOpt: securityOpt,+ Privileged: kit.FromPtr(prepared.current.HostConfig).Privileged,+ },+ NetworkingConfig: func() *network.NetworkingConfig {+ if networkMode == "" || !networkMode.IsUserDefined() {+ return nil+ }+ return &network.NetworkingConfig{+ EndpointsConfig: map[string]*network.EndpointSettings{+ string(networkMode): {Aliases: []string{"arcane-upgrader"}},+ },+ }+ }(),+ Name: upgraderName,+ })})if err != nil {return "", fmt.Errorf("create upgrader container: %w", err)}- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {+ return cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {+ return cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ })return "", fmt.Errorf("start upgrader container: %w", containerStartErr)}diff --git a/backend/internal/system/handler.go b/backend/internal/system/handler.goindex b479d584f..f94820050 100644--- a/backend/internal/system/handler.go+++ b/backend/internal/system/handler.go@@ -1,7 +1,6 @@package systemimport (- "cmp""context""errors""strings"@@ -11,20 +10,15 @@ import ("github.com/getarcaneapp/arcane/types/v2/container""github.com/getarcaneapp/arcane/types/v2/dockerinfo""github.com/getarcaneapp/arcane/types/v2/system"- dockersystem "github.com/moby/moby/api/types/system"- "github.com/moby/moby/client""go.getarcane.app/docker/convert""go.getarcane.app/docker/convert/types"- "go.getarcane.app/sys/cgroup"- "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil")// SystemHandler handles system management endpoints.type SystemHandler struct {- dockerService *docker.DockerClientServicesystemService *SystemServiceappCtx context.Context}@@ -62,19 +56,13 @@ type ConvertDockerRunOutput struct {Body system.ConvertDockerRunResponse}-func NewHandler(dockerService *docker.DockerClientService, systemService *SystemService, appCtx context.Context) *SystemHandler {- return &SystemHandler{dockerService: dockerService, systemService: systemService, appCtx: appCtx}+func NewHandler(systemService *SystemService, appCtx context.Context) *SystemHandler {+ return &SystemHandler{systemService: systemService, appCtx: appCtx}}// Health checks if the Docker daemon is responsive.func (h *SystemHandler) Health(ctx context.Context, input *SystemHealthInput) (*struct{}, error) {- dockerClient, err := h.dockerService.GetClient(ctx)- if err != nil {- return nil, huma.Error503ServiceUnavailable("Failed to connect to Docker: " + err.Error())- }-- _, err = dockerClient.Ping(ctx, client.PingOptions{})- if err != nil {+ if err := h.systemService.PingDocker(ctx); err != nil {return nil, huma.Error503ServiceUnavailable("Docker is not responsive: " + err.Error())}@@ -83,75 +71,12 @@ func (h *SystemHandler) Health(ctx context.Context, input *SystemHealthInput) (*// GetDockerInfo returns Docker daemon version and system information.func (h *SystemHandler) GetDockerInfo(ctx context.Context, input *GetDockerInfoInput) (*GetDockerInfoOutput, error) {- dockerClient, err := h.dockerService.GetClient(ctx)- if err != nil {- return nil, huma.Error500InternalServerError("Failed to connect to Docker: " + err.Error())- }-- version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})- if err != nil {- return nil, huma.Error500InternalServerError("Failed to get Docker version: " + err.Error())- }-- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})+ info, err := h.systemService.GetDockerInfo(ctx)if err != nil {return nil, huma.Error500InternalServerError("Failed to get Docker info: " + err.Error())}- info := infoResult.Info-- // Apply cgroup limits only when running outside Docker (e.g. in LXC).- // In Docker, --cpus/--memory are artificial operator constraints that- // should not cap the host totals shown in the dashboard. The Docker- // daemon's NCPU/MemTotal already reflect the real host. In LXC the- // daemon may report the physical machine's full capacity while the- // LXC guest has a smaller cgroup budget — apply those limits so the- // dashboard shows what Arcane's host actually has available.- if !cgroup.IsDockerContainer() {- if cgroupLimits, detectLimitsErr := cgroup.DetectLimits(); detectLimitsErr == nil {- if limit := cgroupLimits.MemoryLimit; limit > 0 && (info.MemTotal == 0 || limit < info.MemTotal) {- info.MemTotal = limit // keep: copying Docker's ~70-field system.Info as a literal is unreadable- }- if cgroupLimits.CPUCount > 0 && (info.NCPU == 0 || cgroupLimits.CPUCount < info.NCPU) {- info.NCPU = cgroupLimits.CPUCount // keep: copying Docker's ~70-field system.Info as a literal is unreadable- }- }- }-- gitCommit, goVersion, buildTime := extractVersionDetailsFromComponents(version.Components)-- return &GetDockerInfoOutput{- Body: dockerinfo.Info{- Success: true,- APIVersion: version.APIVersion,- GitCommit: gitCommit,- GoVersion: goVersion,- Os: version.Os,- Arch: version.Arch,- BuildTime: buildTime,- Info: info,- },- }, nil-}--func extractVersionDetailsFromComponents(components []dockersystem.ComponentVersion) (gitCommit, goVersion, buildTime string) {- for _, component := range components {- if component.Details == nil {- continue- }-- for key, value := range component.Details {- switch strings.ToLower(key) {- case "gitcommit":- gitCommit = cmp.Or(gitCommit, value)- case "goversion":- goVersion = cmp.Or(goVersion, value)- case "buildtime":- buildTime = cmp.Or(buildTime, value)- }- }- }- return gitCommit, goVersion, buildTime+ return &GetDockerInfoOutput{Body: info}, nil}// StartAllContainers starts all Docker containers.diff --git a/backend/internal/system/module.go b/backend/internal/system/module.goindex f3eb46dd9..65d7e1051 100644--- a/backend/internal/system/module.go+++ b/backend/internal/system/module.go@@ -10,7 +10,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config"- "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup""github.com/getarcaneapp/arcane/backend/v2/internal/system/children/prune"@@ -22,7 +21,6 @@ import (// Module wires the system domain and mounts its routes.type Module struct {service *SystemService- dockerService *docker.DockerClientServiceactivityService *activity.ActivityServicecfg *config.Config}@@ -30,11 +28,10 @@ type Module struct {// New wires system routes around an existing service.func New(service *SystemService,- dockerService *docker.DockerClientService,activityService *activity.ActivityService,cfg *config.Config,) *Module {- return &Module{service: service, dockerService: dockerService, activityService: activityService, cfg: cfg}+ return &Module{service: service, activityService: activityService, cfg: cfg}}// Service exposes the system service to collaborators.@@ -53,7 +50,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont}service := cmp.Or(m.service, &SystemService{})- RegisterSystem(api, NewHandler(m.dockerService, m.service, appCtx.Context()))+ RegisterSystem(api, NewHandler(m.service, appCtx.Context()))prune.RegisterRoutes(api, prune.NewHandler(service.prune, appCtx.Context()))upgrade.RegisterRoutes(api, upgrade.NewHandler(service.upgrade, m.cfg, appCtx.Context()))backup.RegisterRoutes(api, backup.NewHandler(service.backup, m.activityService, appCtx.Context()))diff --git a/backend/internal/system/service.go b/backend/internal/system/service.goindex 287cc739e..119c6c4c4 100644--- a/backend/internal/system/service.go+++ b/backend/internal/system/service.go@@ -14,6 +14,7 @@ import (activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"containertypes "github.com/getarcaneapp/arcane/types/v2/container"+ "github.com/getarcaneapp/arcane/types/v2/dockerinfo""github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/system"usertypes "github.com/getarcaneapp/arcane/types/v2/user"@@ -100,7 +101,7 @@ func NewSystemService(dockerHostMemoryCache: hot.NewHotCache[struct{}, dockerHostMemoryInfo](hot.LRU, 1).WithTTL(dockerHostMemoryCacheTTL).Build(),- prune: prune.NewService(cfg.GetLocation(), dockerService.GetClient, activityService, imageUpdateService, volumeService, networkService),+ prune: prune.NewService(cfg.GetLocation(), dockerService, activityService, imageUpdateService, volumeService, networkService),upgrade: upgrade.NewService(db, dockerService, versionService, eventService, settingsService, projectService, resolveUpgraderRuntimeOptions),}s.backup = systembackup.NewService(systembackup.Dependencies{@@ -302,6 +303,70 @@ func (s *SystemService) performBatchContainerAction(ctx context.Context, environreturn result, listErr}+// PingDocker checks that the Docker daemon is responsive.+func (s *SystemService) PingDocker(ctx context.Context) error {+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.PingResult, error) {+ return cli.Ping(ctx, client.PingOptions{})+ })+ return err+}++// GetDockerInfo returns Docker daemon version and system information.+func (s *SystemService) GetDockerInfo(ctx context.Context) (dockerinfo.Info, error) {+ serverVersion, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ServerVersionResult, error) {+ return cli.ServerVersion(ctx, client.ServerVersionOptions{})+ })+ if err != nil {+ return dockerinfo.Info{}, fmt.Errorf("failed to get Docker version: %w", err)+ }++ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.SystemInfoResult, error) {+ return cli.Info(ctx, client.InfoOptions{})+ })+ if err != nil {+ return dockerinfo.Info{}, err+ }+ info := infoResult.Info++ // Outside Docker (e.g. LXC) the daemon may report the whole machine, so cap totals at the guest's cgroup budget.+ // Inside Docker, --cpus/--memory are operator limits and the daemon's totals already reflect the host.+ if !cgroup.IsDockerContainer() {+ if cgroupLimits, detectLimitsErr := cgroup.DetectLimits(); detectLimitsErr == nil {+ if limit := cgroupLimits.MemoryLimit; limit > 0 && (info.MemTotal == 0 || limit < info.MemTotal) {+ info.MemTotal = limit // keep: copying Docker's ~70-field system.Info as a literal is unreadable+ }+ if cgroupLimits.CPUCount > 0 && (info.NCPU == 0 || cgroupLimits.CPUCount < info.NCPU) {+ info.NCPU = cgroupLimits.CPUCount // keep: copying Docker's ~70-field system.Info as a literal is unreadable+ }+ }+ }++ var gitCommit, goVersion, buildTime string+ for _, component := range serverVersion.Components {+ for key, value := range component.Details {+ switch strings.ToLower(key) {+ case "gitcommit":+ gitCommit = cmp.Or(gitCommit, value)+ case "goversion":+ goVersion = cmp.Or(goVersion, value)+ case "buildtime":+ buildTime = cmp.Or(buildTime, value)+ }+ }+ }++ return dockerinfo.Info{+ Success: true,+ APIVersion: serverVersion.APIVersion,+ GitCommit: gitCommit,+ GoVersion: goVersion,+ Os: serverVersion.Os,+ Arch: serverVersion.Arch,+ BuildTime: buildTime,+ Info: info,+ }, nil+}+func (s *SystemService) GetDiskUsagePath(ctx context.Context) string {return cmp.Or(s.settingsService.GetSettingsConfig().DiskUsagePath.Value, "/")}@@ -347,11 +412,9 @@ func (s *SystemService) loadDockerHostMemory(ctx context.Context) (dockerHostMemif err != nil {return dockerHostMemoryInfo{}, err}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return dockerHostMemoryInfo{}, err- }- inspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ inspect, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerID, client.ContainerInspectOptions{})+ })if err != nil {return dockerHostMemoryInfo{}, err}@@ -364,7 +427,9 @@ func (s *SystemService) loadDockerHostMemory(ctx context.Context) (dockerHostMemreturn dockerHostMemoryInfo{}, err}- info, err := dockerClient.Info(ctx, client.InfoOptions{})+ info, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.SystemInfoResult, error) {+ return cli.Info(ctx, client.InfoOptions{})+ })if err != nil {return dockerHostMemoryInfo{}, err}@@ -423,26 +488,29 @@ const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger,// backupStore persists system backup runs and policies for the backup child.func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) systembackup.Store {+ toModel := func(dto *backuptypes.SystemBackupRun) SystemBackupRun {+ return SystemBackupRun{ //exhaustruct:enforce+ BaseModel: database.BaseModel{+ ID: dto.ID,+ CreatedAt: time.Time{},+ UpdatedAt: nil,+ },+ Size: dto.Size,+ CreatedAt: dto.CreatedAt,+ Status: SystemBackupStatus(dto.Status),+ Trigger: SystemBackupTrigger(dto.Trigger),+ Destination: dto.Destination,+ LocalSnapshotID: dto.LocalSnapshotID,+ RemoteSnapshotID: dto.RemoteSnapshotID,+ S3DestinationID: dto.S3DestinationID,+ S3DestinationName: dto.S3DestinationName,+ PolicyID: dto.PolicyID,+ Error: dto.Error,+ }+ }return systembackup.Store{CreateRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {- run := SystemBackupRun{ //exhaustruct:enforce- BaseModel: database.BaseModel{- ID: dto.ID,- CreatedAt: time.Time{},- UpdatedAt: nil,- },- Size: dto.Size,- CreatedAt: dto.CreatedAt,- Status: SystemBackupStatus(dto.Status),- Trigger: SystemBackupTrigger(dto.Trigger),- Destination: dto.Destination,- LocalSnapshotID: dto.LocalSnapshotID,- RemoteSnapshotID: dto.RemoteSnapshotID,- S3DestinationID: dto.S3DestinationID,- S3DestinationName: dto.S3DestinationName,- PolicyID: dto.PolicyID,- Error: dto.Error,- }+ run := toModel(dto)if err := s.db.WithContext(ctx).Create(&run).Error; err != nil {return err}@@ -465,24 +533,7 @@ func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) sysreturn nil},SaveRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {- return s.db.WithContext(ctx).Save(&SystemBackupRun{ //exhaustruct:enforce- BaseModel: database.BaseModel{- ID: dto.ID,- CreatedAt: time.Time{},- UpdatedAt: nil,- },- Size: dto.Size,- CreatedAt: dto.CreatedAt,- Status: SystemBackupStatus(dto.Status),- Trigger: SystemBackupTrigger(dto.Trigger),- Destination: dto.Destination,- LocalSnapshotID: dto.LocalSnapshotID,- RemoteSnapshotID: dto.RemoteSnapshotID,- S3DestinationID: dto.S3DestinationID,- S3DestinationName: dto.S3DestinationName,- PolicyID: dto.PolicyID,- Error: dto.Error,- }).Error+ return s.db.WithContext(ctx).Save(new(toModel(dto))).Error},FailRun: func(ctx context.Context, id, message string) error {return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("id = ?", id).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": message}).Errordiff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.goindex 6870953a7..87fc69290 100644--- a/backend/internal/updater/children/recovery/service.go+++ b/backend/internal/updater/children/recovery/service.go@@ -8,6 +8,7 @@ import ("slices""strings"+ "github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/scheduler"arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater""github.com/moby/moby/api/types/container"@@ -17,6 +18,7 @@ import ("go.getarcane.app/updater""go.getarcane.app/updater/refs"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref")@@ -27,14 +29,19 @@ var errNoFrozenPlan = errors.New("interrupted update has no frozen target plan")// Service freezes update targets before any pull, pins pulls to the frozen// digests, and confirms or resumes interrupted updates.type Service struct {- dockerClient func(ctx context.Context) (*client.Client, error)- digestResolver func() updater.RegistryDigestResolver+ dockerService *docker.DockerClientService+ images remoteDigesterpendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error)activityID func(ctx context.Context) stringacquireUpdate func(ctx context.Context) (context.Context, func(), error)applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error)}+// remoteDigester resolves registry digests; the image service implements it.+type remoteDigester interface {+ RemoteDigest(ctx context.Context, imageRef string, credentials []containerregistry.Credential) (string, error)+}+type (frozenPendingKey struct{}frozenSingleKey struct{}@@ -50,16 +57,16 @@ type frozenSingle struct {}func NewService(- dockerClient func(ctx context.Context) (*client.Client, error),- digestResolver func() updater.RegistryDigestResolver,+ dockerService *docker.DockerClientService,+ images remoteDigester,pendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error),activityID func(ctx context.Context) string,acquireUpdate func(ctx context.Context) (context.Context, func(), error),applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error),) *Service {return &Service{- dockerClient: dockerClient,- digestResolver: digestResolver,+ dockerService: dockerService,+ images: images,pendingUpdates: pendingUpdates,activityID: activityID,acquireUpdate: acquireUpdate,@@ -123,18 +130,19 @@ func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excludedselected := make([]updater.ImageUpdateRecord, 0, len(records))var targets []arcaneupdater.FrozenUpdateTargetif len(records) > 0 {- dockerClient, clientErr := s.dockerClient(ctx)- if clientErr != nil {- return ctx, clientErr- }- listed, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})+ listed, listErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})+ })if listErr != nil {return ctx, listErr}// Resolve each candidate's configured image once, not per record.candidateRefs := make(map[string]string, len(listed.Items))for _, candidate := range listed.Items {- candidateRefs[candidate.ID] = refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, candidate))+ configured, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {+ return imageref.ConfiguredImage(ctx, dockerClient, candidate), nil+ })+ candidateRefs[candidate.ID] = refs.NormalizeImageUpdateRef(configured)}for _, record := range records {for _, candidate := range listed.Items {@@ -205,11 +213,7 @@ func (s *Service) freezeRecordTarget(ctx context.Context, record updater.ImageUpif pinned := kit.Ternary(record.IsTagUpdate(), "", kit.FromPtr(record.LatestDigest)); pinned != "" {return pinned, nil}- resolver := s.digestResolver()- if resolver == nil {- return "", errors.New("cannot freeze update without registry digest resolver")- }- return resolver.ImageDigest(ctx, record.NewImageRef())+ return s.images.RemoteDigest(ctx, record.NewImageRef(), nil)}()if err != nil {return nil, record, err@@ -247,11 +251,9 @@ func (s *Service) freezeRecordTarget(ctx context.Context, record updater.ImageUp// FreezeContainer captures the identity evidence of one container.func (s *Service) FreezeContainer(ctx context.Context, id string) (*arcaneupdater.FrozenUpdateTarget, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return nil, err- }- result, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, id, client.ContainerInspectOptions{})+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, id, client.ContainerInspectOptions{})+ })if err != nil {return nil, err}@@ -271,11 +273,7 @@ func (s *Service) FreezeContainer(ctx context.Context, id string) (*arcaneupdatefunc (s *Service) PreparePull(ctx context.Context, imageRef string) (string, error) {if single, ok := ctx.Value(frozenSingleKey{}).(*frozenSingle); ok {if single.Target.DesiredImageRef == "" {- resolver := s.digestResolver()- if resolver == nil {- return "", errors.New("cannot freeze update without registry digest resolver")- }- digest, err := resolver.ImageDigest(ctx, imageRef)+ digest, err := s.images.RemoteDigest(ctx, imageRef, nil)if err != nil {return "", err}@@ -334,11 +332,9 @@ func (s *Service) PreparePull(ctx context.Context, imageRef string) (string, err// ConfirmTarget returns whether the desired effect is confirmed,// or whether the exact original container is unchanged and safe to resume.func (s *Service) ConfirmTarget(ctx context.Context, target arcaneupdater.FrozenUpdateTarget) (bool, bool, error) {- dockerClient, err := s.dockerClient(ctx)- if err != nil {- return false, false, err- }- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ listed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if err != nil {return false, false, err}@@ -347,17 +343,21 @@ func (s *Service) ConfirmTarget(ctx context.Context, target arcaneupdater.Frozenif imageErr != nil {return ""}- image, imageInspectErr := dockerClient.ImageInspect(ctx, immutable)+ inspected, imageInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, immutable)+ })if imageInspectErr != nil {return ""}- return image.ID+ return inspected.ID}()for _, candidate := range listed.Items {if !slices.ContainsFunc(candidate.Names, func(name string) bool { return strings.TrimPrefix(name, "/") == target.ContainerName }) {continue}- result, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, candidate.ID, client.ContainerInspectOptions{})+ result, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, candidate.ID, client.ContainerInspectOptions{})+ })if inspectErr != nil {return false, false, inspectErr}diff --git a/backend/internal/updater/children/recovery/service_test.go b/backend/internal/updater/children/recovery/service_test.goindex d76390adf..b4fc46b65 100644--- a/backend/internal/updater/children/recovery/service_test.go+++ b/backend/internal/updater/children/recovery/service_test.go@@ -3,11 +3,13 @@ package recoveryimport ("context""encoding/json/v2"+ "errors""net/http""net/http/httptest""strings""testing"+ "github.com/getarcaneapp/arcane/types/v2/containerregistry""github.com/getarcaneapp/arcane/types/v2/scheduler"arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater""github.com/moby/moby/api/types/container"@@ -17,9 +19,17 @@ import ("github.com/stretchr/testify/require""go.getarcane.app/updater"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext")+// unresolvedDigests fails every registry digest lookup.+type unresolvedDigests struct{}++func (unresolvedDigests) RemoteDigest(context.Context, string, []containerregistry.Credential) (string, error) {+ return "", errors.New("registry digest unavailable")+}+var neverDeferred = func(updater.ImageUpdateRecord, container.Summary) bool { return false }func newTestService(t *testing.T, server *httptest.Server, pending func(context.Context) ([]updater.ImageUpdateRecord, error)) *Service {@@ -28,8 +38,8 @@ func newTestService(t *testing.T, server *httptest.Server, pending func(context.require.NoError(t, err)t.Cleanup(func() { _ = dockerClient.Close() })return NewService(- func(context.Context) (*client.Client, error) { return dockerClient, nil },- func() updater.RegistryDigestResolver { return nil },+ docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),+ unresolvedDigests{},pending,func(context.Context) string { return "" },func(ctx context.Context) (context.Context, func(), error) { return ctx, func() {}, nil },diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.goindex 15bade45a..0ced7dc1b 100644--- a/backend/internal/updater/service.go+++ b/backend/internal/updater/service.go@@ -160,7 +160,6 @@ func NewUpdaterService(upgrade selfUpgradeService,activityService *activity.ActivityService,cfg *config.Config,- _ *runs.Coordinator,admission *runs.Admission,roles *role.RoleService,) (*UpdaterService, error) {@@ -183,8 +182,8 @@ func NewUpdaterService(}// keep: recovery callbacks bind to the allocated serviceservice.recovery = recovery.NewService(- service.DockerClient,- func() updater.RegistryDigestResolver { return registries },+ localDocker,+ imageSvc,service.PendingImageUpdates,activityIDFromContext,service.acquireUpdate,@@ -396,12 +395,11 @@ func (s *UpdaterService) applyScopedUpdates(ctx context.Context, options arcaneureturn docker.ComposeServiceLabel(summary.Labels) != "" && wanted[strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))]}if scope == "image" {- dockerClient, clientErr := s.DockerClient(ctx)- if clientErr != nil {- return nil, clientErr- }matches = func(summary container.Summary) bool {- return wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, summary))]+ configured, _ := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {+ return imageref.ConfiguredImage(ctx, dockerClient, summary), nil+ })+ return wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(configured)]}}var matched []string@@ -646,12 +644,12 @@ func (s *UpdaterService) recordAutoUpdateEvent(ctx context.Context, severity eve}}-// DockerClient returns Arcane's configured Docker client for the updater engine.+// DockerClient implements updater.DockerClientProvider for the updater engine.func (s *UpdaterService) DockerClient(ctx context.Context) (*client.Client, error) {return s.deps.Docker.GetClient(ctx)}-// PullImage pulls an image through Arcane's image service, bounded by the+// PullImage pulls an image through Arcane's image service, which bounds it by the// dockerImagePullTimeout setting so a stuck pull cannot hold the auto-update run.func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progress io.Writer) error {pulledRef, err := s.recovery.PreparePull(ctx, imageRef)@@ -661,33 +659,17 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progreswriter := activitylib.NewWriter(ctx, s.deps.Activity, activityIDFromContext(ctx), progress, "Pulling updated images")defer activitylib.FlushWriter(writer)- pullCtx, cancelPull := context.WithTimeout(ctx, timeouts.GetDuration(s.deps.Settings.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))- defer cancelPull()-- credentials, err := s.deps.Projects.ResolveRegistryCredentials(pullCtx)+ credentials, err := s.deps.Projects.ResolveRegistryCredentials(ctx)if err != nil {return fmt.Errorf("resolve registry credentials: %w", err)}- pull := func(ctx context.Context) error {- return s.deps.ImagePuller.PullImage(ctx, pulledRef, writer, s.deps.SystemUser, credentials)- }- // A frozen digest pull is tagged back to the reference the engine asked for.- if pulledRef != imageRef {- dockerClient, clientErr := s.DockerClient(pullCtx)- if clientErr != nil {- return clientErr- }- if pullErr := imageref.PullAsTag(pullCtx, dockerClient, pulledRef, imageRef, pull); pullErr != nil {- return pullErr- }- } else if pullErr := pull(pullCtx); pullErr != nil {- return pullErr- }- // Reconcile by tag on the run context so a late pull still clears its update records (#4306).- if reconcileErr := s.deps.ImagePuller.ReconcilePulledImageUpdate(ctx, imageRef); reconcileErr != nil {- slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", imageRef, "error", reconcileErr)- }- return nil+ return s.deps.ImagePuller.Pull(ctx, image.PullRequest{+ Ref: pulledRef,+ Tag: imageRef,+ Progress: writer,+ User: s.deps.SystemUser,+ Credentials: credentials,+ })}// PendingImageUpdates returns pending image update records from Arcane's database.@@ -721,17 +703,17 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Imaif scopedCount == 0 {return nil, nil}- dockerClient, clientErr := s.DockerClient(ctx)- if clientErr != nil {- return nil, clientErr- }- listed, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ listed, listErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ })if listErr != nil {return nil, listErr}matched := make([]eligibleContainer, 0, len(listed.Items))for _, cnt := range listed.Items {- configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)+ configured, _ := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {+ return imageref.ConfiguredImage(ctx, dockerClient, cnt), nil+ })resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))if policyErr != nil || resolved.Strategy == "tag" {continue@@ -787,21 +769,23 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record upda}// A shared record stays pending while another non-tag-policy container still runs an older image.- dockerClient, err := s.DockerClient(ctx)- if err != nil {- return err- }- target, err := dockerClient.ImageInspect(ctx, record.NewImageRef())+ target, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, record.NewImageRef())+ })if err != nil {return err}- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ listed, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ })if err != nil {return err}for _, cnt := range listed.Items {// Untagged images list by ID, so an orphaned container matches by its configured image.- configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)+ configured, _ := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {+ return imageref.ConfiguredImage(ctx, dockerClient, cnt), nil+ })resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))if policyErr != nil || resolved.Strategy == "tag" {continue@@ -948,11 +932,9 @@ func (s *UpdaterService) singleContainerActivityRequest(ctx context.Context, conname := func() string {lookupCtx, cancelLookup := context.WithTimeout(ctx, 2*time.Second)defer cancelLookup()- dockerClient, dockerErr := s.DockerClient(lookupCtx)- if dockerErr != nil {- return containerID- }- inspected, inspectErr := compat.ContainerInspectWithCompatibility(lookupCtx, dockerClient, containerID, client.ContainerInspectOptions{})+ inspected, inspectErr := s.deps.Docker.Call(lookupCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})+ })if inspectErr != nil {return containerID}@@ -1186,12 +1168,10 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]stru}func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, out map[string]struct{}) error {- dcli, err := s.DockerClient(ctx)- if err != nil {- return err- }excludedContainers := docker.ExcludedContainerNameSet(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""))- listResult, err := dcli.ContainerList(ctx, client.ContainerListOptions{})+ listResult, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ })if err != nil {return err}@@ -1216,7 +1196,9 @@ func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, ou}// The summary only names an image ID, so the image's tags and the configured reference stand in.- inspectResult, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dcli, summary.ID, client.ContainerInspectOptions{})+ inspectResult, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, summary.ID, client.ContainerInspectOptions{})+ })if inspectErr != nil {slog.DebugContext(ctx, "collectUsedImagesFromContainers: container inspect failed", "containerId", summary.ID, "error", inspectErr)continue@@ -1227,7 +1209,9 @@ func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, ou}if _, inspected := inspectedImages[inspectResult.Container.Image]; !inspected {inspectedImages[inspectResult.Container.Image] = struct{}{}- if imageInspect, imageErr := dcli.ImageInspect(ctx, inspectResult.Container.Image); imageErr == nil {+ if imageInspect, imageErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, inspectResult.Container.Image)+ }); imageErr == nil {for _, tag := range imageInspect.RepoTags {if strings.TrimSpace(tag) != "" && tag != "<none>:<none>" {addNormalizedImageUpdateRef(ctx, out, tag, "normalizedTagsForContainer repo tag", "imageId", inspectResult.Container.Image)@@ -1250,13 +1234,14 @@ func (s *UpdaterService) collectUsedImagesFromProjects(ctx context.Context, outactiveProjectNames := map[string]struct{}{}for _, project := range allProjects {+ name := strings.TrimSpace(project.Name)if project.IsArchived ||(project.Status != projectpkg.ProjectStatusRunning && project.Status != projectpkg.ProjectStatusPartiallyRunning) ||- strings.TrimSpace(project.Name) == "" {+ name == "" {continue}- activeProjectNames[strings.TrimSpace(project.Name)] = struct{}{}- if normalized := projects.NormalizeProjectName(strings.TrimSpace(project.Name)); normalized != "" {+ activeProjectNames[name] = struct{}{}+ if normalized := projects.NormalizeProjectName(name); normalized != "" {activeProjectNames[normalized] = struct{}{}}}@@ -1264,11 +1249,9 @@ func (s *UpdaterService) collectUsedImagesFromProjects(ctx context.Context, outreturn nil}- dockerClient, err := s.deps.Docker.GetClient(ctx)- if err != nil {- return err- }- composeContainers, err := projects.ListGlobalComposeContainers(ctx, dockerClient, s.deps.Docker.DockerHost())+ composeContainers, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {+ return projects.ListGlobalComposeContainers(ctx, dockerClient, s.deps.Docker.DockerHost())+ })if err != nil {return err}diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.goindex 134ddd012..717505fa9 100644--- a/backend/internal/updater/service_test.go+++ b/backend/internal/updater/service_test.go@@ -269,7 +269,6 @@ func TestUpdaterService_ApplyPendingNoRecords(t *testing.T) {nil,nil,nil,- nil,admission,nil,)@@ -335,7 +334,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("server label triggers upgrade with system user", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{@@ -356,7 +355,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("engine target keeps the selected tag and pulls the frozen digest", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)require.NoError(t, svcErr)frozen := svc.recovery.WithSingleTarget(ctx, &arcaneupdater.FrozenUpdateTarget{@@ -379,7 +378,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("agent label triggers upgrade", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane-agent", map[string]string{@@ -392,7 +391,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "demo", map[string]string{@@ -406,7 +405,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("upgrade errors are wrapped", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{triggerError: errors.New("upgrade failed")}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{@@ -419,7 +418,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {}func TestUpdaterService_StatusTracking(t *testing.T) {- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)stopContainer := svc.BeginContainerUpdate("container-1")@@ -445,7 +444,7 @@ func TestUpdaterService_DockerClientAdapter(t *testing.T) {ctx := t.Context()t.Run("active engine updates correlate before the stop event is recorded", func(t *testing.T) {events := event.NewEventService(nil, nil, nil)- svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)end := svc.engine.BeginContainerUpdate("updating-container")require.True(t, events.ShouldSuppressDaemonEvent("container", "updating-container", "", ""))@@ -459,7 +458,7 @@ func TestUpdaterService_DockerClientAdapter(t *testing.T) {server := newImagePullServerWithObserver(t, nil, nil)wantClient := newTestDockerClient(t, server)dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(wantClient)- svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)gotClient, err := svc.DockerClient(ctx)@@ -480,10 +479,19 @@ func TestUpdaterService_PullImageAdapter(t *testing.T) {t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })server := newImagePullServerWithObserver(t, nil, nil)dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))- imageSvc := image.NewImageService(db, dockerSvc, nil, nil, nil, event.NewEventService(db, nil, nil), nil, nil)+ imageSvc := image.NewImageService(+ db,+ dockerSvc,+ nil,+ imageupdate.NewImageUpdateService(db, settingsService, nil, dockerSvc, nil, nil, nil),+ nil,+ event.NewEventService(db, nil, nil),+ settingsService,+ nil,+ )projectSvc := project.NewProjectService(db, nil, nil, nil, testDockerService, nil, nil, nil, nil, nil, nil)projectSvc.RegistryCredentialsProvider = environment.NewEnvironmentService(db, nil, nil, nil, nil, nil).GetEnabledRegistryCredentials- svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)var progress bytes.Buffer@@ -509,16 +517,25 @@ func TestUpdaterService_PullImageAdapter(t *testing.T) {gotAuth = authHeader}})+ settingsService, err := settings.NewSettingsService(ctx, db)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))registrySvc := registry.NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)- imageSvc := image.NewImageService(db, dockerSvc, registrySvc, nil, nil, event.NewEventService(db, nil, nil), nil, nil)+ imageSvc := image.NewImageService(+ db,+ dockerSvc,+ registrySvc,+ imageupdate.NewImageUpdateService(db, nil, registrySvc, dockerSvc, nil, nil, nil),+ nil,+ event.NewEventService(db, nil, nil),+ settingsService,+ nil,+ )envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)projectSvc := project.NewProjectService(db, nil, nil, nil, testDockerService, nil, nil, nil, nil, nil, nil)projectSvc.RegistryCredentialsProvider = envSvc.GetEnabledRegistryCredentials- settingsService, err := settings.NewSettingsService(ctx, db)- require.NoError(t, err)- t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })- svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)var progress bytes.Buffer@@ -565,7 +582,7 @@ func TestUpdaterService_PendingImageUpdatesAdapter(t *testing.T) {require.NoError(t, db.AutoMigrate(¬ification.NotificationSettings{}))notif := notification.NewNotificationService(db, nil, nil, nil, nil)imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)require.NoError(t, svcErr)records, err := svc.PendingImageUpdates(ctx)@@ -610,8 +627,12 @@ func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnredb := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, ¬ification.NotificationSettings{}))notif := notification.NewNotificationService(db, nil, nil, nil, nil)- imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)- svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ }))+ t.Cleanup(server.Close)+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, notif, nil)+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)require.NoError(t, err)require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{ID: "sha256:pending",@@ -655,7 +676,7 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotifications(t *testinnotif := notification.NewNotificationService(db, nil, nil, nil, nil)imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)require.NoError(t, svcErr)require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{@@ -686,7 +707,7 @@ func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotified(t *testinnotif := notification.NewNotificationService(db, nil, nil, nil, nil)imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)require.NoError(t, svcErr)require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{@@ -710,7 +731,7 @@ func TestUpdaterService_RecordUpdateRunAdapter(t *testing.T) {ctx := t.Context()db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}))- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)err := svc.RecordUpdateRun(ctx, updater.ResourceResult{@@ -756,7 +777,7 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessage(t *testing.T) {db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}, &activity.Activity{}, &activity.ActivityMessage{}))activityService := activity.NewActivityService(db, nil)- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil)require.NoError(t, svcErr)for _, tt := range []struct {@@ -841,7 +862,7 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivity(t *testingharness := flowtest.New(t, activityService)admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())require.NoError(t, admission.Register(harness.Runtime))- svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, admission, nil)+ svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, admission, nil)require.NoError(t, err)require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))harness.Start(t)@@ -894,7 +915,7 @@ func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizes(t *testing.Tdb := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))activityService := activity.NewActivityService(db, nil)- svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)+ svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil)require.NoError(t, err)// A host that has not started cannot start the workflow.require.NoError(t, svc.execution.RegisterWorkflows(flowtest.New(t, activityService).Engine))@@ -982,7 +1003,7 @@ func TestUpdaterService_SingleContainerActivityResultMessages(t *testing.T) {db := setupProjectTestDB(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))activityService := activity.NewActivityService(db, nil)- svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)+ svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil)require.NoError(t, err)started, err := activityService.StartActivity(t.Context(),@@ -1128,7 +1149,6 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjects(t *nil,nil,nil,- nil,admission,nil,)@@ -1282,7 +1302,6 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgrade(mockUpgrade,nil,nil,- nil,admission,nil,)@@ -1555,7 +1574,7 @@ func TestUpdaterService_ScopedPendingRecordsIsolation(t *testing.T) {require.NoError(t, db.AutoMigrate(¬ification.NotificationSettings{}))notif := notification.NewNotificationService(db, nil, nil, nil, nil)imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, dockerSvc, nil, notif, nil)- svc, err := NewUpdaterService(db, nil, dockerSvc, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)require.NoError(t, err)records, err := svc.PendingImageUpdates(t.Context())require.NoError(t, err)@@ -1632,7 +1651,7 @@ func TestUpdaterService_SharedDigestRecordClearing(t *testing.T) {}server := newUpdaterApplyPendingDockerServer(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))- svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)require.NoError(t, svc.ClearImageUpdateRecord(t.Context(), updater.ImageUpdateRecord{ContainerID: "updated",@@ -1783,7 +1802,6 @@ func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {nil,activityService,nil,- nil,admission,nil,)@@ -1806,7 +1824,7 @@ func TestUpdaterService_WorkflowDefinitionsAreVersioned(t *testing.T) {harness := flowtest.New(t, activity.NewActivityService(db, nil))imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)require.NoError(t, imageUpdates.RegisterWorkflows(harness.Engine))- svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)require.NoError(t, svc.RegisterWorkflows(harness.Engine))harness.Start(t)@@ -1845,7 +1863,7 @@ func TestUpdaterService_AutoUpdateDeferral(t *testing.T) {require.NoError(t, err)t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })require.NoError(t, settingsService.SetIntSetting(ctx, "autoUpdateDelay", 24))- svc, err := NewUpdaterService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)+ svc, err := NewUpdaterService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, err)deferred, err := svc.autoUpdateDeferral(ctx, now)diff --git a/backend/internal/version/service.go b/backend/internal/version/service.goindex c97f90b04..7e74ebcc7 100644--- a/backend/internal/version/service.go+++ b/backend/internal/version/service.go@@ -12,7 +12,6 @@ import ("github.com/distribution/reference""github.com/getarcaneapp/arcane/types/v2/version"- containertypes "github.com/moby/moby/api/types/container""github.com/moby/moby/client""github.com/samber/hot""github.com/samber/mo"@@ -84,7 +83,7 @@ func NewVersionService(loader := func(_ []struct{}) (map[struct{}]latestRelease, error) {ctx, cancel := context.WithTimeout(context.Background(), defaultRequestTimeout) //nolint:forbidigo // Cache revalidation runs independently of request cancellation.defer cancel()- release, err := service.fetchLatestReleaseInternal(ctx)+ release, err := service.fetchLatestRelease(ctx)if err != nil {return nil, err}@@ -99,7 +98,7 @@ func NewVersionService(return service}-func (s *VersionService) getLatestReleaseInternal(_ context.Context) (latestRelease, error) {+func (s *VersionService) getLatestRelease(_ context.Context) (latestRelease, error) {release, found, err := s.cache.Get(struct{}{})if err != nil {return latestRelease{}, err@@ -110,7 +109,7 @@ func (s *VersionService) getLatestReleaseInternal(_ context.Context) (latestRelereturn release, nil}-func (s *VersionService) fetchLatestReleaseInternal(ctx context.Context) (latestRelease, error) {+func (s *VersionService) fetchLatestRelease(ctx context.Context) (latestRelease, error) {req, err := http.NewRequestWithContext(ctx, http.MethodGet, versionCheckURL, http.NoBody)if err != nil {return latestRelease{}, fmt.Errorf("create GitHub request: %w", err)@@ -146,7 +145,7 @@ func (s *VersionService) fetchLatestReleaseInternal(ctx context.Context) (latest}func (s *VersionService) GetLatestVersion(ctx context.Context) (string, error) {- rel, err := s.getLatestReleaseInternal(ctx)+ rel, err := s.getLatestRelease(ctx)if err != nil {return "", err}@@ -203,57 +202,41 @@ func (s *VersionService) GetVersionInformation(ctx context.Context, currentVersi}cur := s.normalizeVersion(currentVersion)- check := &version.Check{- CurrentVersion: cur,- ReleaseURL: s.ReleaseURL(""),- UpdateAvailable: false,- }-- if s.disabled {- return check, nil- }-- latest, err := s.GetLatestVersion(ctx)- if err != nil {- return check, err- }-+ latest, err := func() (string, error) {+ if s.disabled {+ return "", nil+ }+ return s.GetLatestVersion(ctx)+ }()// Skip when latest is semver-older than current (e.g. a prerelease build// asking the stable release feed) — a downgrade is not a newest version.- if latest != "" && !s.IsNewer(cur, latest) {- check.NewestVersion = latest- check.UpdateAvailable = s.IsNewer(latest, cur)- check.ReleaseURL = s.ReleaseURL(latest)- }-- return check, nil+ newest := kit.Ternary(err == nil && latest != "" && !s.IsNewer(cur, latest), latest, "")+ return &version.Check{+ CurrentVersion: cur,+ NewestVersion: newest,+ UpdateAvailable: newest != "" && s.IsNewer(newest, cur),+ ReleaseURL: s.ReleaseURL(newest),+ }, err}-// isNextBuildInternal reports whether this build tracks the next channel: a+// isNextBuild reports whether this build tracks the next channel: a// -next. prerelease version, or a container running the rolling next image tag.-func (s *VersionService) isNextBuildInternal(currentTag string) bool {+func (s *VersionService) isNextBuild(currentTag string) bool {return strings.Contains(strings.ToLower(strings.TrimSpace(s.version)), "-next.") ||strings.TrimSpace(currentTag) == "next"}-// resolveNextVersionInternal returns the normalized version label of the image+// resolveNextVersion returns the normalized version label of the image// an upgrade would pull, or "" when it cannot be resolved. It prefers the// digest reference so the reported version and NewestDigest describe the same// artifact.-func (s *VersionService) resolveNextVersionInternal(ctx context.Context, imageRef, tag, newestDigest string) string {- if s.containerRegistryService == nil || strings.TrimSpace(imageRef) == "" {+func (s *VersionService) resolveNextVersion(ctx context.Context, imageRef, tag, newestDigest string) string {+ newestDigest, tag = strings.TrimSpace(newestDigest), strings.TrimSpace(tag)+ if s.containerRegistryService == nil || strings.TrimSpace(imageRef) == "" || (newestDigest == "" && tag == "") {return ""}- var lookupRef string- switch {- case strings.TrimSpace(newestDigest) != "":- lookupRef = imageRef + "@" + strings.TrimSpace(newestDigest)- case strings.TrimSpace(tag) != "":- lookupRef = imageRef + ":" + strings.TrimSpace(tag)- default:- return ""- }+ lookupRef := kit.Ternary(newestDigest != "", imageRef+"@"+newestDigest, imageRef+":"+tag)label, err := s.containerRegistryService.ImageVersionLabel(ctx, lookupRef)if err != nil {@@ -269,20 +252,8 @@ func (s *VersionService) resolveNextVersionInternal(ctx context.Context, imageRereturn normalized}-// isSemverVersion checks if a version string is semver-based (e.g., v1.0.0)-func (s *VersionService) isSemverVersion() bool {- return semver.IsValid(kit.EnsurePrefix(s.version, "v"))-}--// getDisplayVersion formats the version for display purposes-// Semver versions (including prereleases like 2.4.0-next.1) display as v<version>-func (s *VersionService) getDisplayVersion() string {- versionValue := strings.TrimPrefix(strings.TrimSpace(s.version), "v")- return kit.Ternary(s.isSemverVersion(), "v"+versionValue, versionValue)-}--// updateCheckImageRefInternal points the digest lookup at the configured registry; "auto" keeps the running reference.-func (s *VersionService) updateCheckImageRefInternal(currentImageRef string) string {+// updateCheckImageRef points the digest lookup at the configured registry; "auto" keeps the running reference.+func (s *VersionService) updateCheckImageRef(currentImageRef string) string {if s.settingsService == nil {return currentImageRef}@@ -312,18 +283,66 @@ func (s *VersionService) updateCheckImageRefInternal(currentImageRef string) str// GetAppVersionInfo returns application version information including display versionfunc (s *VersionService) GetAppVersionInfo(ctx context.Context) *version.Info {- isSemver := s.isSemverVersion()+ isSemver := semver.IsValid(kit.EnsurePrefix(s.version, "v"))ver := s.normalizeVersion(s.version)+ // Semver versions (including prereleases like 2.4.0-next.1) display as v<version>.+ displayVersion := strings.TrimPrefix(strings.TrimSpace(s.version), "v")// Always detect current image infocurrentTag, currentDigest, currentImageRef, currentImageID := s.detectCurrentImageInfo(ctx)- // Build base info struct (always populated)- info := &version.Info{+ // Update fields stay empty when update checks are disabled.+ update := func() version.Info {+ if s.disabled {+ return version.Info{}+ }+ checkImageRef := s.updateCheckImageRef(currentImageRef)+ digestUpdateAvailable, latestDigest := s.storedOrDigestBasedUpdate(ctx, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef)++ switch {+ case isSemver && s.isNextBuild(currentTag):+ nextVersion := s.resolveNextVersion(ctx, checkImageRef, currentTag, latestDigest)+ return version.Info{+ NewestDigest: latestDigest,+ NewestVersion: kit.Ternary(nextVersion != "" && semver.Compare(nextVersion, ver) >= 0, nextVersion, ""),+ UpdateAvailable: digestUpdateAvailable || (nextVersion != "" && s.IsNewer(nextVersion, ver)),+ }+ case isSemver:+ // Never surface a release older than the running version as the target.+ rel, err := s.getLatestRelease(ctx)+ if err != nil || rel.TagName == "" || s.IsNewer(ver, rel.TagName) {+ return version.Info{NewestDigest: latestDigest}+ }+ return version.Info{+ NewestDigest: latestDigest,+ NewestVersion: rel.TagName,+ UpdateAvailable: s.IsNewer(rel.TagName, ver),+ ReleaseURL: s.ReleaseURL(rel.TagName),+ ReleaseNotes: rel.Body,+ ReleasedAt: rel.PublishedAt,+ }+ }++ // Best-effort: pull release notes for non-semver track too, so the modal can preview+ // the latest tagged release even when the running build is digest-tracking.+ rel, err := s.getLatestRelease(ctx)+ if err != nil || rel.TagName == "" {+ return version.Info{NewestDigest: latestDigest, UpdateAvailable: digestUpdateAvailable}+ }+ return version.Info{+ NewestDigest: latestDigest,+ UpdateAvailable: digestUpdateAvailable,+ ReleaseURL: s.ReleaseURL(rel.TagName),+ ReleaseNotes: rel.Body,+ ReleasedAt: rel.PublishedAt,+ }+ }()++ return &version.Info{CurrentVersion: ver,CurrentTag: currentTag,CurrentDigest: currentDigest,- DisplayVersion: s.getDisplayVersion(),+ DisplayVersion: kit.Ternary(isSemver, "v"+displayVersion, displayVersion),Revision: s.revision,ShortRevision: config.ShortRevision(),GoVersion: config.GoVersion(),@@ -332,55 +351,17 @@ func (s *VersionService) GetAppVersionInfo(ctx context.Context) *version.Info {EnabledFeatures: append(kit.Unique(kit.TrimNonEmpty(strings.Split(strings.ToLower(buildables.EnabledFeatures), ","))), apns.FeatureName),BuildTime: config.BuildTime,IsSemverVersion: isSemver,- UpdateAvailable: false,- }-- // If update checks disabled, return base info- if s.disabled {- return info- }-- checkImageRef := s.updateCheckImageRefInternal(currentImageRef)- digestUpdateAvailable, latestDigest := s.storedOrDigestBasedUpdateInternal(ctx, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef)- if latestDigest != "" {- info.NewestDigest = latestDigest- }-- switch {- case isSemver && s.isNextBuildInternal(currentTag):- nextVersion := s.resolveNextVersionInternal(ctx, checkImageRef, currentTag, latestDigest)- if nextVersion != "" && semver.Compare(nextVersion, ver) >= 0 {- info.NewestVersion = nextVersion- }- info.UpdateAvailable = digestUpdateAvailable || (nextVersion != "" && s.IsNewer(nextVersion, ver))- case isSemver:- // Never surface a release older than the running version as the target.- rel, err := s.getLatestReleaseInternal(ctx)- if err == nil && rel.TagName != "" && !s.IsNewer(ver, rel.TagName) {- info.NewestVersion = rel.TagName- info.UpdateAvailable = s.IsNewer(rel.TagName, ver)- info.ReleaseURL = s.ReleaseURL(rel.TagName)- info.ReleaseNotes = rel.Body- info.ReleasedAt = rel.PublishedAt- }- default:- // Best-effort: pull release notes for non-semver track too, so the modal can preview- // the latest tagged release even when the running build is digest-tracking.- if rel, err := s.getLatestReleaseInternal(ctx); err == nil && rel.TagName != "" {- info.ReleaseNotes = rel.Body- info.ReleasedAt = rel.PublishedAt- if info.ReleaseURL == "" {- info.ReleaseURL = s.ReleaseURL(rel.TagName)- }- }- info.UpdateAvailable = digestUpdateAvailable+ NewestVersion: update.NewestVersion,+ NewestDigest: update.NewestDigest,+ UpdateAvailable: update.UpdateAvailable,+ ReleaseURL: update.ReleaseURL,+ ReleaseNotes: update.ReleaseNotes,+ ReleasedAt: update.ReleasedAt,}-- return info}-// storedOrDigestBasedUpdateInternal uses the stored poller row only when the check targets the running reference.-func (s *VersionService) storedOrDigestBasedUpdateInternal(ctx context.Context, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef string) (bool, string) {+// storedOrDigestBasedUpdate uses the stored poller row only when the check targets the running reference.+func (s *VersionService) storedOrDigestBasedUpdate(ctx context.Context, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef string) (bool, string) {if s.imageUpdateService != nil && strings.TrimSpace(currentImageID) != "" && checkImageRef == currentImageRef {record, found, err := s.imageUpdateService.StoredUpdateByImageID(ctx, currentImageID)if err != nil {@@ -390,11 +371,21 @@ func (s *VersionService) storedOrDigestBasedUpdateInternal(ctx context.Context,}}- if currentTag != "" && currentDigest != "" && checkImageRef != "" && s.containerRegistryService != nil {- return s.checkDigestBasedUpdate(ctx, currentTag, currentDigest, checkImageRef)+ if currentTag == "" || currentDigest == "" || checkImageRef == "" || s.containerRegistryService == nil {+ return false, ""+ }++ latestDigest, err := s.containerRegistryService.ImageDigest(ctx, checkImageRef+":"+currentTag)+ if err != nil {+ slog.WarnContext(ctx, "Failed to fetch latest digest for tag", "tag", currentTag, "error", err)+ return false, ""}- return false, ""+ updateAvailable := currentDigest != latestDigest && latestDigest != ""+ if updateAvailable {+ slog.InfoContext(ctx, "Digest-based update available", "tag", currentTag, "currentDigest", currentDigest, "latestDigest", latestDigest)+ }+ return updateAvailable, latestDigest}// detectCurrentImageInfo attempts to detect the current container's image tag and digest@@ -410,130 +401,56 @@ func (s *VersionService) detectCurrentImageInfo(ctx context.Context) (tag, digesreturn "", "", "", ""}- containerId := s.detectContainerID(ctx, dockerClient)+ containerId, err := cgroup.CurrentContainerID()+ if err != nil {+ // Fall back to the Arcane label when cgroup/hostname detection fails.+ slog.DebugContext(ctx, "detectCurrentImageInfo: cgroup detection failed, trying label fallback", "error", err)+ containerId = libarcane.FindArcaneContainerIDByLabel(ctx, dockerClient)+ }if containerId == "" {slog.DebugContext(ctx, "detectCurrentImageInfo: could not detect container ID")return "", "", "", ""}slog.DebugContext(ctx, "detectCurrentImageInfo: detected container", "containerId", containerId)- inspectResult, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerId, client.ContainerInspectOptions{})+ inspectResult, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerId, client.ContainerInspectOptions{})+ })if err != nil {slog.DebugContext(ctx, "detectCurrentImageInfo: failed to inspect container", "containerId", containerId, "error", err)return "", "", "", ""}- container := inspectResult.Container- imageID = container.Image-- configImage := ""- if container.Config != nil {- configImage = container.Config.Image- }-- // Parse tag from container config image (user-specified reference)- tag = s.extractTagFromImageRef(configImage)-- // Get digest and normalized imageRef from container image- imageRef, digest = s.extractImageDetails(ctx, dockerClient, container)-- // Fallback to container config image if RepoDigests didn't provide imageRef- if imageRef == "" {- imageRef = s.normalizeImageRef(configImage)- }-- return tag, digest, imageRef, imageID-}+ imageID = inspectResult.Container.Image+ configImage := kit.FromPtr(inspectResult.Container.Config).Image-// detectContainerID tries to get the current container ID, falling back to label-based detection-func (s *VersionService) detectContainerID(ctx context.Context, dockerClient *client.Client) string {- containerId, err := s.getCurrentContainerID()- if err == nil {- slog.DebugContext(ctx, "detectContainerID: found via getCurrentContainerID", "containerId", containerId)- return containerId+ // The tag comes from the user-specified config image; the repository falls back to it too.+ tag, imageRef = "latest", configImage+ if named, parseErr := reference.ParseNormalizedNamed(configImage); parseErr == nil {+ imageRef = named.Name()+ if tagged, ok := named.(reference.Tagged); ok {+ tag = tagged.Tag()+ }}- slog.DebugContext(ctx, "detectContainerID: getCurrentContainerID failed, trying label fallback", "error", err)-- // Fallback: locate the Arcane container by label (works even when cgroup/hostname detection fails)- return libarcane.FindArcaneContainerIDByLabel(ctx, dockerClient)-}-// extractImageDetails extracts digest and imageRef from a container's image-func (s *VersionService) extractImageDetails(ctx context.Context, dockerClient *client.Client, container containertypes.InspectResponse) (imageRef, digest string) {- if container.Image == "" {- return "", ""+ if imageID == "" {+ return tag, "", imageRef, imageID}-- imageInspect, err := dockerClient.ImageInspect(ctx, container.Image)+ imageInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {+ return cli.ImageInspect(ctx, imageID)+ })if err != nil {- return "", ""+ return tag, "", imageRef, imageID}-- // Extract digest and repository from first RepoDigest using reference libraryfor _, repoDigest := range imageInspect.RepoDigests {- named, parseNormalizedNamedErr := reference.ParseNormalizedNamed(repoDigest)- if parseNormalizedNamedErr != nil {+ named, parseErr := reference.ParseNormalizedNamed(repoDigest)+ if parseErr != nil {continue}if digested, ok := named.(reference.Digested); ok {- return named.Name(), string(digested.Digest())+ return tag, string(digested.Digest()), named.Name(), imageID}}-- return "", ""-}--// normalizeImageRef extracts just the repository name from an image reference-func (s *VersionService) normalizeImageRef(configImage string) string {- if named, err := reference.ParseNormalizedNamed(configImage); err == nil {- return named.Name()- }- return configImage-}--// getCurrentContainerID detects if we're running in Docker via cgroup, mountinfo, or hostname-func (s *VersionService) getCurrentContainerID() (string, error) {- return cgroup.CurrentContainerID()-}--// extractTagFromImageRef extracts the tag from an image reference using distribution/reference-func (s *VersionService) extractTagFromImageRef(imageRef string) string {- named, err := reference.ParseNormalizedNamed(imageRef)- if err != nil {- return "latest"- }-- tagged, ok := named.(reference.Tagged)- if ok {- return tagged.Tag()- }-- return "latest"-}--// checkDigestBasedUpdate checks if there's a newer digest for the current tag-func (s *VersionService) checkDigestBasedUpdate(ctx context.Context, currentTag, currentDigest, currentImageRef string) (updateAvailable bool, latestDigest string) {- if currentTag == "" || currentDigest == "" || currentImageRef == "" {- return false, ""- }-- // Build full image reference with tag- imageRef := fmt.Sprintf("%s:%s", currentImageRef, currentTag)-- // Fetch latest digest from registry- latestDigest, err := s.containerRegistryService.ImageDigest(ctx, imageRef)- if err != nil {- slog.WarnContext(ctx, "Failed to fetch latest digest for tag", "tag", currentTag, "error", err)- return false, ""- }-- // Compare digests - if they differ, an update is available- updateAvailable = currentDigest != latestDigest && latestDigest != ""-- if updateAvailable {- slog.InfoContext(ctx, "Digest-based update available", "tag", currentTag, "currentDigest", currentDigest, "latestDigest", latestDigest)- }-- return updateAvailable, latestDigest+ return tag, "", imageRef, imageID}// RunStreamProducer shares one version poll loop across every connected client and emits a snapshot whenever it changes.diff --git a/backend/internal/version/service_test.go b/backend/internal/version/service_test.goindex e3644ee18..4e092dd6e 100644--- a/backend/internal/version/service_test.go+++ b/backend/internal/version/service_test.go@@ -35,7 +35,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/registry")-func TestVersionService_GetAppVersionInfoDoesNotUseStoredDigestUpdateForSemverBuildInternal(t *testing.T) {+func TestVersionService_GetAppVersionInfoDoesNotUseStoredDigestUpdateForSemverBuild(t *testing.T) {ctx := t.Context()db := setupImageUpdateTestDB(t)@@ -117,7 +117,7 @@ func TestVersionService_GetAppVersionInfoDoesNotUseStoredDigestUpdateForSemverBuRequest: req,}, nil})}- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)svc := NewVersionService(httpClient, false, "1.2.3", "revision", nil, dockerService, imageUpdateService, nil)@@ -140,7 +140,7 @@ func TestVersionService_GetAppVersionInfoDisplaysSemverNextVersion(t *testing.T)assert.True(t, info.IsSemverVersion)}-func TestVersionService_GetAppVersionInfoPreservesDevVersionInternal(t *testing.T) {+func TestVersionService_GetAppVersionInfoPreservesDevVersion(t *testing.T) {svc := NewVersionService(nil, true, "dev", "unknown", nil, nil, nil, nil)info := svc.GetAppVersionInfo(t.Context())@@ -169,8 +169,8 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {return f(req)}-// newTestDockerClientInternal builds a Docker client pointed at a fake Docker HTTP server.-func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {+// newTestDockerClient builds a Docker client pointed at a fake Docker HTTP server.+func newTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {t.Helper()httpClient := server.Client()@@ -188,7 +188,7 @@ func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.return cli}-func TestVersionService_IsNextBuildInternal(t *testing.T) {+func TestVersionService_IsNextBuild(t *testing.T) {tests := []struct {name stringversion string@@ -203,14 +203,14 @@ func TestVersionService_IsNextBuildInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {svc := NewVersionService(nil, true, tt.version, "rev", nil, nil, nil, nil)- assert.Equal(t, tt.want, svc.isNextBuildInternal(tt.currentTag))+ assert.Equal(t, tt.want, svc.isNextBuild(tt.currentTag))})}}-// newLabelRegistryTestServerInternal starts an in-memory OCI registry and+// newLabelRegistryTestServer starts an in-memory OCI registry and// returns its host as "localhost:PORT" so go-containerregistry uses plain HTTP.-func newLabelRegistryTestServerInternal(t *testing.T) string {+func newLabelRegistryTestServer(t *testing.T) string {t.Helper()server := httptest.NewServer(ggcrregistry.New(ggcrregistry.Logger(log.New(io.Discard, "", 0))))t.Cleanup(server.Close)@@ -219,9 +219,9 @@ func newLabelRegistryTestServerInternal(t *testing.T) string {return "localhost:" + serverURL.Port()}-// newNextChannelDockerServerInternal fakes a Docker daemon whose Arcane+// newNextChannelDockerServer fakes a Docker daemon whose Arcane// container runs repo:tag with the given current repo digest.-func newNextChannelDockerServerInternal(t *testing.T, containerID, imageID, repo, tag, currentDigest string) *httptest.Server {+func newNextChannelDockerServer(t *testing.T, containerID, imageID, repo, tag, currentDigest string) *httptest.Server {t.Helper()imageRef := repo + ":" + tagserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -256,7 +256,7 @@ func newNextChannelDockerServerInternal(t *testing.T, containerID, imageID, reporeturn server}-func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *testing.T) {+func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabel(t *testing.T) {ctx := t.Context()db := setupImageUpdateTestDB(t)@@ -264,7 +264,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *tecontainerID = "arcane-container-1234567890"imageID = "sha256:arcane-image")- registryHost := newLabelRegistryTestServerInternal(t)+ registryHost := newLabelRegistryTestServer(t)repo := registryHost + "/getarcaneapp/arcane"img, err := random.Image(64, 1)@@ -282,7 +282,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *terequire.NoError(t, err)currentDigest := digest.FromString("current-arcane").String()- server := newNextChannelDockerServerInternal(t, containerID, imageID, repo, "next", currentDigest)+ server := newNextChannelDockerServer(t, containerID, imageID, repo, "next", currentDigest)require.NoError(t, db.WithContext(ctx).Create(&imageupdate.ImageUpdateRecord{ID: imageID,@@ -306,7 +306,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *teRequest: req,}, nil})}- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)registrySvc := registry.NewContainerRegistryService(nil, nil, nil, nil)svc := NewVersionService(httpClient, false, "2.8.0-next.65", "revision", registrySvc, dockerService, imageUpdateService, nil)@@ -322,7 +322,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *teassert.False(t, githubHit, "next channel must not consult the GitHub stable release feed")}-func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigestInternal(t *testing.T) {+func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigest(t *testing.T) {ctx := t.Context()db := setupImageUpdateTestDB(t)@@ -331,12 +331,12 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigesimageID = "sha256:arcane-image")// Registry with no image pushed: the label fetch fails.- registryHost := newLabelRegistryTestServerInternal(t)+ registryHost := newLabelRegistryTestServer(t)repo := registryHost + "/getarcaneapp/arcane"currentDigest := digest.FromString("current-arcane").String()latestDigest := digest.FromString("latest-arcane").String()- server := newNextChannelDockerServerInternal(t, containerID, imageID, repo, "next", currentDigest)+ server := newNextChannelDockerServer(t, containerID, imageID, repo, "next", currentDigest)require.NoError(t, db.WithContext(ctx).Create(&imageupdate.ImageUpdateRecord{ID: imageID,@@ -350,7 +350,7 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigesCheckTime: time.Now().UTC(),}).Error)- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)registrySvc := registry.NewContainerRegistryService(nil, nil, nil, nil)svc := NewVersionService(nil, false, "2.8.0-next.65", "revision", registrySvc, dockerService, imageUpdateService, nil)@@ -363,7 +363,7 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigesassert.Equal(t, latestDigest, info.NewestDigest)}-func TestVersionService_GetAppVersionInfoSuppressesOlderStableForPrereleaseInternal(t *testing.T) {+func TestVersionService_GetAppVersionInfoSuppressesOlderStableForPrerelease(t *testing.T) {httpClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {return &http.Response{StatusCode: http.StatusOK,diff --git a/backend/internal/volume/children/backup/children/restore/service.go b/backend/internal/volume/children/backup/children/restore/service.goindex 923cc863e..f61e9f8ae 100644--- a/backend/internal/volume/children/backup/children/restore/service.go+++ b/backend/internal/volume/children/backup/children/restore/service.go@@ -20,6 +20,7 @@ import ("github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/mount""github.com/moby/moby/client"+ "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/backup""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -31,7 +32,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser")-const restoreBackupFilesScriptInternal = `set -e+const restoreBackupFilesScript = `set -earchive="$1"shiftarchive_mode=$(stat -c '%A' -- "$archive" 2>/dev/null) || { echo ARCANE_NOT_FOUND >&2; exit 44; }@@ -50,12 +51,12 @@ type Dependencies struct {Run func(ctx context.Context, backupID string) (*volume.Backup, error)Repository func(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) (backup.Repository, string, error)Password func(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error)- StopContainers func(ctx context.Context, dockerClient *client.Client, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error)- StartContainers func(ctx context.Context, dockerClient *client.Client, stopped []container.Summary, user usertypes.Actor) ([]container.Summary, error)+ StopContainers func(ctx context.Context, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error)+ StartContainers func(ctx context.Context, stopped []container.Summary, user usertypes.Actor) ([]container.Summary, error)SafetyBackup func(ctx context.Context, volumeName string, user usertypes.Actor) (string, error)ArchivePaths func(ctx context.Context, backupID string) ([]string, error)ArchiveFilename func(backupID string) (string, error)- HelperImage func(ctx context.Context, dockerClient *client.Client) (string, error)+ HelperImage func(ctx context.Context) (string, error)StorageMount func(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error)AcquireHelper func(ctx context.Context, volumeName string) (string, func(), error)Exec func(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error)@@ -96,11 +97,11 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID stringreturn err}}- stopped, err := s.deps.StopContainers(ctx, dockerClient, volumeName, user, true)+ stopped, err := s.deps.StopContainers(ctx, volumeName, user, true)containersStopped := len(stopped) > 0defer func() {if containersStopped {- _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)+ _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)err = errors.Join(err, restartErr)}}()@@ -110,11 +111,15 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string// A discovered backup may target a volume missing on this instance; create it and skip the safety backup of an empty volume.preBackupID := ""var createdVolume bool- if _, inspectErr := dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{}); inspectErr != nil {+ if _, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{})+ }); inspectErr != nil {if !errdefs.IsNotFound(inspectErr) {return fmt.Errorf("failed to inspect volume for restore: %w", inspectErr)}- if _, createErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: volumeName}); createErr != nil {+ if _, createErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: volumeName})+ }); createErr != nil {return fmt.Errorf("failed to create missing volume for restore: %w", createErr)}createdVolume = true@@ -126,7 +131,7 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string}}if entry.Format == volume.BackupFormatArchive {- if restoreArchiveBackupErr := s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID); restoreArchiveBackupErr != nil {+ if restoreArchiveBackupErr := s.restoreArchiveBackup(ctx, dockerClient, volumeName, backupID, nil); restoreArchiveBackupErr != nil {return restoreArchiveBackupErr}} else if restoreSnapshotErr := s.deps.Engine.RestoreSnapshot(@@ -147,7 +152,7 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID stringreturn fmt.Errorf("failed to restore Rustic snapshot: %w", restoreSnapshotErr)}if containersStopped {- stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)+ stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)containersStopped = len(stopped) > 0if err != nil {return err@@ -163,57 +168,50 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID stringreturn nil}-type selectionInternal struct {+type selectionPlan struct {entries []backuptypes.BackupFileEntryarchivePaths []stringglobalRoot bool}-func (- s *Service,-) resolveRestoreSelectionInternal(+func (s *Service) resolveRestoreSelection(ctx context.Context,dockerClient *client.Client,entry *volume.Backup,repository backup.Repository,snapshotID string,selection backuptypes.RestoreSelection,-) (- selectionInternal,- error,-) {+) (selectionPlan, error) {if selection.SelectAll && strings.TrimSpace(selection.Search) == "" {if _, err := backupbrowser.NormalizeSelection(selection, []backuptypes.BackupFileEntry{{Path: "", IsDirectory: true}}); err != nil {- return selectionInternal{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)+ return selectionPlan{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)}- return selectionInternal{globalRoot: true}, nil+ return selectionPlan{globalRoot: true}, nil}- resolved := selectionInternal{}- var eligible []backuptypes.BackupFileEntry- var err error- if entry.Format == volume.BackupFormatArchive {- resolved.archivePaths, err = s.deps.ArchivePaths(ctx, entry.ID)- eligible = backupbrowser.BuildEntries(resolved.archivePaths, "", true)- } else {- var listed []string+ paths, err := func() ([]string, error) {+ if entry.Format == volume.BackupFormatArchive {+ return s.deps.ArchivePaths(ctx, entry.ID)+ }password, passwordErr := s.deps.Password(ctx, dockerClient, repository)if passwordErr != nil {- return selectionInternal{}, passwordErr+ return nil, passwordErr}- listed, err = s.deps.Engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)- eligible = backupbrowser.BuildEntries(listed, "", true)- }+ return s.deps.Engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)+ }()if err != nil {- return selectionInternal{}, err+ return selectionPlan{}, err}- resolved.entries, err = backupbrowser.NormalizeSelection(selection, eligible)+ entries, err := backupbrowser.NormalizeSelection(selection, backupbrowser.BuildEntries(paths, "", true))if err != nil {- return selectionInternal{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)+ return selectionPlan{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)}- return resolved, nil+ return selectionPlan{+ entries: entries,+ archivePaths: kit.Ternary(entry.Format == volume.BackupFormatArchive, paths, nil),+ }, nil}-func archiveMembersForSelectionInternal(paths []string, selected []backuptypes.BackupFileEntry) []string {+func archiveMembersForSelection(paths []string, selected []backuptypes.BackupFileEntry) []string {files := make([]string, 0, len(paths))for _, raw := range paths {directory := strings.HasSuffix(strings.TrimSpace(raw), "/")@@ -235,23 +233,18 @@ func archiveMembersForSelectionInternal(paths []string, selected []backuptypes.Breturn slices.Compact(files)}-func (- s *Service,-) restoreSelectionInternal(+func (s *Service) restoreSelection(ctx context.Context,dockerClient *client.Client,volumeName, backupID string,entry *volume.Backup,repository backup.Repository,snapshotID string,- selection selectionInternal,+ selection selectionPlan,) error {if entry.Format == volume.BackupFormatArchive {- if selection.globalRoot {- return s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID)- }- members := archiveMembersForSelectionInternal(selection.archivePaths, selection.entries)- return s.restoreArchiveBackupFilesInternal(ctx, dockerClient, volumeName, backupID, members)+ // nil members restores the whole archive.+ return s.restoreArchiveBackup(ctx, dockerClient, volumeName, backupID, kit.Ternary(selection.globalRoot, nil, archiveMembersForSelection(selection.archivePaths, selection.entries)))}password, err := s.deps.Password(ctx, dockerClient, repository)if err != nil {@@ -299,15 +292,15 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID sreturn err}}- resolvedSelection, err := s.resolveRestoreSelectionInternal(ctx, dockerClient, entry, repository, snapshotID, selection)+ resolvedSelection, err := s.resolveRestoreSelection(ctx, dockerClient, entry, repository, snapshotID, selection)if err != nil {return err}- stopped, err := s.deps.StopContainers(ctx, dockerClient, volumeName, user, true)+ stopped, err := s.deps.StopContainers(ctx, volumeName, user, true)containersStopped := len(stopped) > 0defer func() {if containersStopped {- _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)+ _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)err = errors.Join(err, restartErr)}}()@@ -318,7 +311,7 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID sif err != nil {return fmt.Errorf("failed to create pre-restore backup: %w", err)}- if restoreSelectionErr := s.restoreSelectionInternal(+ if restoreSelectionErr := s.restoreSelection(ctx,dockerClient,volumeName,@@ -331,7 +324,7 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID sreturn restoreSelectionErr}if containersStopped {- stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)+ stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)containersStopped = len(stopped) > 0if err != nil {return err@@ -353,12 +346,13 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID sreturn nil}-func (s *Service) restoreArchiveBackupInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string) error {+// restoreArchiveBackup restores a legacy archive into volumeName: all of it when members is nil, otherwise only members.+func (s *Service) restoreArchiveBackup(ctx context.Context, dockerClient *client.Client, volumeName, backupID string, members []string) error {filename, err := s.deps.ArchiveFilename(backupID)if err != nil {return err}- helperImage, err := s.deps.HelperImage(ctx, dockerClient)+ helperImage, err := s.deps.HelperImage(ctx)if err != nil {return err}@@ -366,37 +360,52 @@ func (s *Service) restoreArchiveBackupInternal(ctx context.Context, dockerClientif err != nil {return err}- config := &container.Config{- Image: helperImage,- Cmd: []string{- "sh",- "-c",- fmt.Sprintf(- "set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf "+- "/backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; "+- "find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"",- filename,- filename,- ),- },- Labels: volumehelper.Labels(),- }- hostConfig := volumehelper.HostConfig(helperImage, []string{- volumeName + ":/volume",- }, []mount.Mount{backupMount})- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,+ // A full restore runs the replace script as the container command; a partial one execs into a sleeping helper.+ fullRestoreCmd := []string{+ "sh",+ "-c",+ fmt.Sprintf(+ "set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf "++ "/backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; "++ "find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"",+ filename,+ filename,+ ),+ }+ resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &container.Config{+ Image: helperImage,+ Cmd: kit.Ternary(members == nil, fullRestoreCmd, []string{"sleep", "infinity"}),+ NetworkDisabled: true,+ Labels: volumehelper.Labels(),+ },+ HostConfig: volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume"}, []mount.Mount{backupMount}),+ })})if err != nil {return fmt.Errorf("failed to create restore container: %w", err)}defer func() {- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ })}()- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {+ if _, containerStartErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {return fmt.Errorf("failed to start restore container: %w", containerStartErr)}+ if members != nil {+ stderr, restoreErr := s.RestoreArchiveMembers(ctx, resp.ID, filename, members)+ if restoreErr != nil {+ return restoreErr+ }+ if strings.TrimSpace(stderr) != "" {+ slog.DebugContext(ctx, "volume service: restore files stderr", "backupId", backupID, "stderr", strings.TrimSpace(stderr))+ }+ return nil+ }waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})var waitBody container.WaitResponseselect {@@ -416,7 +425,7 @@ func (s *Service) restoreArchiveBackupInternal(ctx context.Context, dockerClient// container's /volume; the container must mount backup storage at /backups.func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error) {args := make([]string, 0, len(cleanedPaths)+5)- args = append(args, "sh", "-c", restoreBackupFilesScriptInternal, "sh", path.Join("/backups", filename))+ args = append(args, "sh", "-c", restoreBackupFilesScript, "sh", path.Join("/backups", filename))for _, cleaned := range cleanedPaths {args = append(args, "./"+cleaned)}@@ -427,51 +436,6 @@ func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filenareturn stderr, nil}-func (s *Service) restoreArchiveBackupFilesInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string, cleanedPaths []string) error {- filename, err := s.deps.ArchiveFilename(backupID)- if err != nil {- return err- }- helperImage, err := s.deps.HelperImage(ctx, dockerClient)- if err != nil {- return err- }- backupMount, err := s.deps.StorageMount(ctx, dockerClient, "/backups", true)- if err != nil {- return err- }- config := &container.Config{- Image: helperImage,- Cmd: []string{"sleep", "infinity"},- NetworkDisabled: true,- Labels: volumehelper.Labels(),- }- hostConfig := volumehelper.HostConfig(helperImage, []string{- volumeName + ":/volume",- }, []mount.Mount{backupMount})- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,- })- if err != nil {- return fmt.Errorf("failed to create restore container: %w", err)- }- defer func() {- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())- }()- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {- return fmt.Errorf("failed to start restore container: %w", containerStartErr)- }- stderr, err := s.RestoreArchiveMembers(ctx, resp.ID, filename, cleanedPaths)- if err != nil {- return fmt.Errorf("failed to restore files: %w", err)- }- if strings.TrimSpace(stderr) != "" {- slog.DebugContext(ctx, "volume service: restore files stderr", "backupId", backupID, "stderr", strings.TrimSpace(stderr))- }- return nil-}-func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user usertypes.Actor) error {slog.DebugContext(ctx, "volume service: upload and restore", "volume", volumeName, "filename", filename, "user", user.ID)diff --git a/backend/internal/volume/children/backup/children/restore/service_test.go b/backend/internal/volume/children/backup/children/restore/service_test.goindex 8a6c70938..936e85029 100644--- a/backend/internal/volume/children/backup/children/restore/service_test.go+++ b/backend/internal/volume/children/backup/children/restore/service_test.go@@ -8,13 +8,13 @@ import ("github.com/stretchr/testify/require")-func TestArchiveMembersForSelectionExpandsFoldersInternal(t *testing.T) {+func TestArchiveMembersForSelectionExpandsFolders(t *testing.T) {paths := []string{"./folder/", "./folder/a.txt", "./folder/nested/b.txt", "./other.txt"}selected := []backup.BackupFileEntry{{Path: "folder", Name: "folder", IsDirectory: true}}require.Equal(t,[]string{"folder/", "folder/a.txt", "folder/nested/b.txt"},- archiveMembersForSelectionInternal(paths, selected),+ archiveMembersForSelection(paths, selected),)}@@ -36,10 +36,10 @@ func TestRestoreBackupFilesScriptUsesSupportedTooling(t *testing.T) {"tar -r","$((",} {- require.NotContains(t, restoreBackupFilesScriptInternal, unsupported)+ require.NotContains(t, restoreBackupFilesScript, unsupported)}if shellPath, err := exec.LookPath("sh"); err == nil {- output, syntaxErr := exec.Command(shellPath, "-n", "-c", restoreBackupFilesScriptInternal).CombinedOutput()+ output, syntaxErr := exec.Command(shellPath, "-n", "-c", restoreBackupFilesScript).CombinedOutput()require.NoErrorf(t, syntaxErr, "restore script has invalid syntax: %s", output)}}diff --git a/backend/internal/volume/children/backup/helpers.go b/backend/internal/volume/children/backup/helpers.goindex 8dd6fc422..4f1ee4a90 100644--- a/backend/internal/volume/children/backup/helpers.go+++ b/backend/internal/volume/children/backup/helpers.go@@ -56,9 +56,8 @@ func resolveBackupStorageMountFromMounts(ctx context.Context, mounts []containerreturn mirroredMount}-// localRusticRepositoryInitialized reports whether the local volume repository-// has a Rustic config file. Only a /backups mount in the Arcane container can be-// checked without starting Rustic, so any other setup counts as initialized.+// localRusticRepositoryInitialized reports whether the local repository has a Rustic config.+// Only a /backups mount in the Arcane container can be checked without Rustic; other setups count as initialized.func (s *Service) localRusticRepositoryInitialized(ctx context.Context, dockerClient *client.Client) bool {inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)if err != nil || docker.MountForDestination(inspect.Mounts, "/backups", "/backups") == nil {@@ -78,12 +77,8 @@ func backupMountWarningFromArcaneMounts(mounts []container.MountPoint) string {}func (s *Service) backupMountWarning(ctx context.Context) string {- dockerClient, err := s.deps.Docker.GetClient(ctx)- if err != nil {- return ""- }// Cannot determine Arcane mount status (e.g. running outside Docker); suppress warning.- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ inspect, err := s.deps.Docker.Call(ctx, libarcane.InspectCurrentArcaneContainer)if err != nil {return ""}@@ -102,23 +97,25 @@ func (s *Service) StorageMount(ctx context.Context, dockerClient *client.Client,}}slog.DebugContext(ctx, "volume service: ensure backup volume", "backupVolume", s.deps.BackupVolumeName)- ensureClient, err := s.deps.Docker.GetClient(ctx)- if err != nil {- return mount.Mount{}, err- }- if _, inspectErr := ensureClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{}); inspectErr != nil {- if _, createErr := ensureClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName}); createErr != nil {+ if _, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{})+ }); inspectErr != nil {+ if _, createErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName})+ }); createErr != nil {return mount.Mount{}, fmt.Errorf("failed to create backup volume: %w", createErr)}}return mount.Mount{Type: mount.TypeVolume, Source: s.deps.BackupVolumeName, Target: target, ReadOnly: readOnly}, nil}-func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClient *client.Client, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error) {+func (s *Service) stopRunningContainersForBackup(ctx context.Context, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error) {if s.deps.StopContainer == nil || s.deps.StartContainer == nil {return nil, errors.New("container service is unavailable")}- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ containers, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list running containers before volume backup: %w", err)}@@ -151,7 +148,7 @@ func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerCliecontinue}if stopContainerErr := s.deps.StopContainer(ctx, candidate.ID, user); stopContainerErr != nil {- stillStopped, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)+ stillStopped, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), stopped, user)return stillStopped, errors.Join(fmt.Errorf("failed to stop container %s before volume backup: %w", candidate.ID, stopContainerErr), restartErr)}stopped = append(stopped, candidate)@@ -160,14 +157,16 @@ func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClie}//nolint:gocognit // recovery retries must reconcile IDs, names, and Compose identities in one bounded loop-func (s *Service) startContainersAfterBackup(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {+func (s *Service) startContainersAfterBackup(ctx context.Context, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {recoveryCtx, cancel := context.WithTimeout(ctx, volumeBackupContainerRecoveryTimeout)defer cancel()remaining := append([]container.Summary(nil), stoppedContainers...)lastErrors := make(map[string]error, len(stoppedContainers))for len(remaining) > 0 {- currentContainers, listErr := dockerClient.ContainerList(recoveryCtx, client.ContainerListOptions{All: true})+ currentContainers, listErr := s.deps.Docker.Call(recoveryCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if listErr == nil {currentByID := make(map[string]container.Summary, len(currentContainers.Items))currentByName := make(map[string]container.Summary, len(currentContainers.Items))@@ -355,24 +354,32 @@ func (s *Service) logBackupEvent(ctx context.Context, eventType event.EventType,}}-func (s *Service) createBackupTempContainerWithMount(ctx context.Context, dockerClient *client.Client, backupMount mount.Mount) (string, func(), error) {- helperImage, err := s.deps.HelperImage(ctx, dockerClient)+func (s *Service) createBackupTempContainerWithMount(ctx context.Context, backupMount mount.Mount) (string, func(), error) {+ helperImage, err := s.deps.HelperImage(ctx)if err != nil {return "", nil, err}- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()},- HostConfig: volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount}),+ resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()},+ HostConfig: volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount}),+ })})if err != nil {return "", nil, fmt.Errorf("failed to create backup temp container: %w", err)}- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ if _, containerStartErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {+ _, _ = s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ })return "", nil, fmt.Errorf("failed to start backup temp container: %w", containerStartErr)}return resp.ID, func() {- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ })}, nil}@@ -388,5 +395,5 @@ func (s *Service) createBackupTempContainer(ctx context.Context, dockerClient *cif err != nil {return "", nil, err}- return s.createBackupTempContainerWithMount(ctx, dockerClient, backupMount)+ return s.createBackupTempContainerWithMount(ctx, backupMount)}diff --git a/backend/internal/volume/children/backup/helpers_test.go b/backend/internal/volume/children/backup/helpers_test.goindex 200f8e645..10777c43e 100644--- a/backend/internal/volume/children/backup/helpers_test.go+++ b/backend/internal/volume/children/backup/helpers_test.go@@ -165,7 +165,7 @@ func TestBackupMountWarningFromArcaneMounts(t *testing.T) {}}-func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) (*Service, *client.Client) {+func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) *Service {t.Helper()server := httptest.NewServer(handler)@@ -180,8 +180,12 @@ func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) (*Servicdb := &database.DB{DB: gormDB}dockerService := docker.NewDockerClientService(t.Context(), db, &config.Config{}, nil).WithClient(dockerClient)eventService := event.NewEventService(db, &config.Config{}, nil)- containerService := containerdomain.NewContainerService(eventService, dockerService, nil, nil, nil)- return NewService(Dependencies{StopContainer: containerService.StopContainer, StartContainer: containerService.StartContainer}), dockerClient+ containerService := containerdomain.NewContainerService(eventService, dockerService, nil, nil, nil, nil)+ return NewService(Dependencies{+ Docker: dockerService,+ StopContainer: containerService.StopContainer,+ StartContainer: containerService.StartContainer,+ })}func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsingVolume(t *testing.T) {@@ -211,14 +215,14 @@ func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsin}})- service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)+ service := setupVolumeBackupLifecycleTest(t, serverHandler)actor := user.Actor{ID: "user-1", Username: "tester"}- stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)+ stopped, err := service.stopRunningContainersForBackup(t.Context(), "app-data", actor, false)require.NoError(t, err)require.Len(t, stopped, 1)require.Equal(t, "uses-volume", stopped[0].ID)- remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)+ remaining, err := service.startContainersAfterBackup(t.Context(), stopped, actor)require.NoError(t, err)require.Empty(t, remaining)require.Equal(t, []string{"stop:uses-volume", "start:uses-volume"}, operations)@@ -255,9 +259,9 @@ func TestVolumeBackupContainerLifecycleRollsBackStoppedContainersOnStopFailure(t}})- service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)+ service := setupVolumeBackupLifecycleTest(t, serverHandler)actor := user.Actor{ID: "user-1", Username: "tester"}- stillStopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)+ stillStopped, err := service.stopRunningContainersForBackup(t.Context(), "app-data", actor, false)require.ErrorContains(t, err, "failed to stop container second")require.Empty(t, stillStopped)require.Equal(t, []string{"stop:first", "stop:second", "start:first"}, operations)@@ -315,14 +319,14 @@ func TestVolumeBackupContainerLifecycleWaitsForRunningComposeReplacement(t *test}})- service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)+ service := setupVolumeBackupLifecycleTest(t, serverHandler)actor := user.Actor{ID: "user-1", Username: "tester"}- stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)+ stopped, err := service.stopRunningContainersForBackup(t.Context(), "app-data", actor, false)require.NoError(t, err)require.Len(t, stopped, 1)require.Equal(t, "old-id", stopped[0].ID)- remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)+ remaining, err := service.startContainersAfterBackup(t.Context(), stopped, actor)require.NoError(t, err)require.Empty(t, remaining)require.Equal(t, []string{"stop:old-id"}, operations)diff --git a/backend/internal/volume/children/backup/service.go b/backend/internal/volume/children/backup/service.goindex b26856b2f..37b14cfec 100644--- a/backend/internal/volume/children/backup/service.go+++ b/backend/internal/volume/children/backup/service.go@@ -77,7 +77,7 @@ type Dependencies struct {AlreadyRunning errorStopContainer func(ctx context.Context, containerID string, user usertypes.Actor) errorStartContainer func(ctx context.Context, containerID string, user usertypes.Actor) error- HelperImage func(ctx context.Context, dockerClient *client.Client) (string, error)+ HelperImage func(ctx context.Context) (string, error)AcquireHelper func(ctx context.Context, volumeName string) (string, func(), error)Exec func(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error)BackupVolumeName string@@ -335,11 +335,16 @@ func (s *Service) CreateSystemManagedBackup(ctx context.Context, volumeName stri}trigger = cmp.Or(trigger, backuptypes.VolumeBackupTriggerManual)transient := &policies.VolumeBackupPolicy{- VolumeName: volumeName, Enabled: true, Schedule: policy.Schedule, RetentionCount: policy.RetentionCount,- StopContainers: policy.StopContainers, LocalEnabled: policy.LocalEnabled, S3Enabled: policy.S3Enabled,+ ID: policyID,+ VolumeName: volumeName,+ Enabled: true,+ Schedule: policy.Schedule,+ RetentionCount: policy.RetentionCount,+ StopContainers: policy.StopContainers,+ LocalEnabled: policy.LocalEnabled,+ S3Enabled: policy.S3Enabled,S3DestinationID: policy.S3DestinationID,}- transient.ID = policyIDplan, err := s.resolveBackupPlan(ctx, volumeName, trigger, volume.CreateBackupRequest{}, transient)if err != nil {return nil, err@@ -359,12 +364,17 @@ func (s *Service) prepareBackup(ctx context.Context, volumeName, trigger, policyreturn nil, nil, err}entry := &volume.Backup{- VolumeName: volumeName, CreatedAt: time.Now(), Status: backuptypes.VolumeBackupStatusRunning,- Trigger: trigger, Destination: backupDestination(plan.LocalEnabled, plan.S3Enabled), Format: volume.BackupFormatRustic,- S3DestinationID: plan.S3DestinationID, PolicyID: policyID,- Type: backuptypes.ManagementTypeForPolicy(policyID),+ ID: fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8]),+ VolumeName: volumeName,+ CreatedAt: time.Now(),+ Status: backuptypes.VolumeBackupStatusRunning,+ Trigger: trigger,+ Destination: backupDestination(plan.LocalEnabled, plan.S3Enabled),+ Format: volume.BackupFormatRustic,+ S3DestinationID: plan.S3DestinationID,+ PolicyID: policyID,+ Type: backuptypes.ManagementTypeForPolicy(policyID),}- entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])if createBackupErr := s.deps.Store.Create(ctx, entry); createBackupErr != nil {lease.Release(ctx)return nil, nil, createBackupErr@@ -420,10 +430,10 @@ func (s *Service) executeBackup(ctx context.Context, entry *volume.Backup, user// stopped holds the containers still awaiting a restart.var stopped []container.Summaryif plan.Policy != nil && plan.Policy.StopContainers {- stopped, err = s.stopRunningContainersForBackup(ctx, dockerClient, volumeName, user, false)+ stopped, err = s.stopRunningContainersForBackup(ctx, volumeName, user, false)defer func() {if len(stopped) > 0 {- _, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)+ _, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), stopped, user)err = errors.Join(err, restartErr)}}()@@ -435,7 +445,7 @@ func (s *Service) executeBackup(ctx context.Context, entry *volume.Backup, userreturn snapshotErr}if len(stopped) > 0 {- if stopped, err = s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user); err != nil {+ if stopped, err = s.startContainersAfterBackup(context.WithoutCancel(ctx), stopped, user); err != nil {return err}}@@ -741,11 +751,15 @@ func (s *Service) DownloadBackup(ctx context.Context, backupID string, user *usereturn nil, 0, passwordErr}scratchVolume := "arcane-rustic-download-" + uuid.New().String()- if _, volumeCreateErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); volumeCreateErr != nil {+ if _, volumeCreateErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()})+ }); volumeCreateErr != nil {return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", volumeCreateErr)}removeScratch := func() {- _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.VolumeRemoveResult, error) {+ return dockerClient.VolumeRemove(ctx, scratchVolume, client.VolumeRemoveOptions{Force: true})+ })}scratchMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}restoreOptions := backup.RestoreOptions{DeleteExtra: true}@@ -754,7 +768,7 @@ func (s *Service) DownloadBackup(ctx context.Context, backupID string, user *usereturn nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", restoreErr)}var removeContainer func()- if containerID, removeContainer, err = s.createBackupTempContainerWithMount(ctx, dockerClient, scratchMount); err != nil {+ if containerID, removeContainer, err = s.createBackupTempContainerWithMount(ctx, scratchMount); err != nil {removeScratch()return nil, 0, err}diff --git a/backend/internal/volume/children/workspace/service.go b/backend/internal/volume/children/workspace/service.goindex 78d89ac89..2f8c33dbb 100644--- a/backend/internal/volume/children/workspace/service.go+++ b/backend/internal/volume/children/workspace/service.go@@ -37,7 +37,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database"dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/event"- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"acfsutils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/acfs"@@ -52,7 +51,7 @@ type Dependencies struct {AcquireHelper func(ctx context.Context, volumeName string) (string, func(), error)RequireACFS func(ctx context.Context, volumeName, containerID string) errorExec func(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error)- HelperImage func(ctx context.Context, dockerClient *client.Client) (string, error)+ HelperImage func(ctx context.Context) (string, error)BackupStorageMount func(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error)ArchiveFileTarget func(ctx context.Context, volumeName, backupID, rel string) (string, string, error)RestoreArchiveMembers func(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error)@@ -72,7 +71,7 @@ func NewService(deps Dependencies) *Service {// MaxFileSizeBytes is the largest file the workspace reads or accepts.func (s *Service) MaxFileSizeBytes() int64 {- return s.volumeWorkspaceMaxFileSizeBytesInternal()+ return s.volumeWorkspaceMaxFileSizeBytes()}func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*workspace.Workspace, error) {@@ -84,8 +83,8 @@ func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*w}()inspectionStartedAt := time.Now()- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {- return nil, classifyVolumeWorkspaceHelperSupportErrorInternal(err)+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {+ return nil, classifyVolumeWorkspaceHelperSupportError(err)}slog.DebugContext(ctx, "volume workspace inspection completed", "volume", volumeName, "duration", time.Since(inspectionStartedAt))@@ -101,7 +100,7 @@ func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*wslog.DebugContext(ctx, "volume workspace helper acquired", "volume", volumeName, "containerId", containerID, "duration", time.Since(helperStartedAt))scanStartedAt := time.Now()- tree, err := s.readVolumeWorkspaceFromContainerInternal(ctx, containerID)+ tree, err := s.readVolumeWorkspaceFromContainer(ctx, containerID)if err != nil {return nil, err}@@ -124,7 +123,7 @@ func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*wreturn tree, nil}-func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context, containerID string) (*workspace.Workspace, error) {+func (s *Service) readVolumeWorkspaceFromContainer(ctx context.Context, containerID string) (*workspace.Workspace, error) {maxDepth := s.deps.MaxDepthif maxDepth <= 0 {maxDepth = 50@@ -163,7 +162,7 @@ func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context,done <- execErr}()- tree, parseErr := decodeVolumeWorkspaceWalkInternal(pipeReader, maxEntries, s.volumeWorkspaceMaxFileSizeBytesInternal())+ tree, parseErr := decodeVolumeWorkspaceWalk(pipeReader, maxEntries, s.volumeWorkspaceMaxFileSizeBytes())if parseErr != nil {cancel()_ = pipeReader.CloseWithError(parseErr)@@ -171,7 +170,7 @@ func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context,execErr := <-done_ = pipeReader.Close()if execErr != nil && (parseErr == nil || ctx.Err() != nil || !errors.Is(execErr, context.Canceled)) {- return nil, classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace")+ return nil, classifyVolumeWorkspaceExecError(execErr, stderr.String(), "read volume workspace")}if parseErr != nil {return nil, fmt.Errorf("parse volume workspace: %w", parseErr)@@ -180,7 +179,7 @@ func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context,return tree, nil}-func (s *Service) volumeWorkspaceMaxFileSizeBytesInternal() int64 {+func (s *Service) volumeWorkspaceMaxFileSizeBytes() int64 {if s.deps.MaxFileSizeBytes <= 0 {return workspacepkg.MaxFileSizeBytes(workspacepkg.DefaultMaxFileSizeMB)}@@ -190,8 +189,8 @@ func (s *Service) volumeWorkspaceMaxFileSizeBytesInternal() int64 {func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relativePath string) (*workspace.FileContent, error) {defer s.deps.Locks.RLock(volumeName)()- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {- return nil, classifyVolumeWorkspaceHelperSupportErrorInternal(err)+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {+ return nil, classifyVolumeWorkspaceHelperSupportError(err)}rel, err := kit.NormalizeRelativePath(relativePath)if err != nil {@@ -209,7 +208,7 @@ func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relati"acfs", "stat", "--root", "/volume", "--path", "/" + rel,})if err != nil {- return nil, classifyVolumeWorkspaceExecErrorInternal(err, stderr, "inspect volume workspace file")+ return nil, classifyVolumeWorkspaceExecError(err, stderr, "inspect volume workspace file")}var statResponse types.StatResponseif unmarshalErr := json.Unmarshal([]byte(stdout), &statResponse); unmarshalErr != nil {@@ -226,18 +225,18 @@ func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relatireturn nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, errors.New("path is a directory"))}if !acfsutils.IsRegular(entry) {- return volumeWorkspaceFileContentResponseInternal(rel, "special", entry.Size, nil, s.volumeWorkspaceMaxFileSizeBytesInternal())+ return volumeWorkspaceFileContentResponse(rel, "special", entry.Size, nil, s.volumeWorkspaceMaxFileSizeBytes())}- maxFileSizeBytes := s.volumeWorkspaceMaxFileSizeBytesInternal()+ maxFileSizeBytes := s.volumeWorkspaceMaxFileSizeBytes()if entry.Size > maxFileSizeBytes {- return volumeWorkspaceFileContentResponseInternal(rel, "regular", entry.Size, nil, maxFileSizeBytes)+ return volumeWorkspaceFileContentResponse(rel, "regular", entry.Size, nil, maxFileSizeBytes)}previewLimit := maxFileSizeBytesif previewLimit < int64(^uint64(0)>>1) {previewLimit++}- reader, size, err := s.startVolumeWorkspaceReadInternal(ctx, containerID, rel, previewLimit, func() {})+ reader, size, err := s.startVolumeWorkspaceRead(ctx, containerID, rel, previewLimit, func() {})if err != nil {return nil, fmt.Errorf("read volume workspace file: %w", err)}@@ -250,16 +249,16 @@ func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relatireturn nil, fmt.Errorf("close volume workspace file content: %w", closeErr)}if size > maxFileSizeBytes {- return volumeWorkspaceFileContentResponseInternal(rel, "regular", size, nil, maxFileSizeBytes)+ return volumeWorkspaceFileContentResponse(rel, "regular", size, nil, maxFileSizeBytes)}- return volumeWorkspaceFileContentResponseInternal(rel, "regular", size, content, maxFileSizeBytes)+ return volumeWorkspaceFileContentResponse(rel, "regular", size, content, maxFileSizeBytes)}func (s *Service) DownloadVolumeWorkspaceFile(ctx context.Context, volumeName, relativePath string) (io.ReadCloser, int64, error) {unlock := s.deps.Locks.RLock(volumeName)- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {unlock()- return nil, 0, classifyVolumeWorkspaceHelperSupportErrorInternal(err)+ return nil, 0, classifyVolumeWorkspaceHelperSupportError(err)}rel, err := kit.NormalizeRelativePath(relativePath)if err != nil {@@ -279,10 +278,10 @@ func (s *Service) DownloadVolumeWorkspaceFile(ctx context.Context, volumeName, rcleanupAndUnlock()return nil, 0, requireVolumeHelperACFSErr}- return s.startVolumeWorkspaceReadInternal(ctx, containerID, rel, 0, cleanupAndUnlock)+ return s.startVolumeWorkspaceRead(ctx, containerID, rel, 0, cleanupAndUnlock)}-type volumeWorkspaceReadStreamInternal struct {+type volumeWorkspaceReadStream struct {ctx context.Contextcancel context.CancelFuncstopClose func() bool@@ -294,7 +293,7 @@ type volumeWorkspaceReadStreamInternal struct {err error}-func (r *volumeWorkspaceReadStreamInternal) Read(buffer []byte) (int, error) {+func (r *volumeWorkspaceReadStream) Read(buffer []byte) (int, error) {if r.remaining == 0 {var extra [1]byteread, readErr := r.pipe.Read(extra[:])@@ -321,7 +320,7 @@ func (r *volumeWorkspaceReadStreamInternal) Read(buffer []byte) (int, error) {return read, err}-func (r *volumeWorkspaceReadStreamInternal) Close() error {+func (r *volumeWorkspaceReadStream) Close() error {r.once.Do(func() {r.stopClose()r.cancel()@@ -335,7 +334,7 @@ func (r *volumeWorkspaceReadStreamInternal) Close() error {return r.err}-func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containerID, relativePath string, limit int64, cleanup func()) (io.ReadCloser, int64, error) {+func (s *Service) startVolumeWorkspaceRead(ctx context.Context, containerID, relativePath string, limit int64, cleanup func()) (io.ReadCloser, int64, error) {dockerClient, err := s.deps.Docker.GetClient(ctx)if err != nil {cleanup()@@ -361,7 +360,7 @@ func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containeexecErr = fmt.Errorf("acfs read exited with code %d", exitCode)}if execErr != nil {- execErr = classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace file")+ execErr = classifyVolumeWorkspaceExecError(execErr, stderr.String(), "read volume workspace file")}stopClose()_ = pipeWriter.CloseWithError(execErr)@@ -389,7 +388,7 @@ func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containereturn nil, 0, errors.New("volume workspace file is too large")}size := int64(payloadSize)- return &volumeWorkspaceReadStreamInternal{+ return &volumeWorkspaceReadStream{ctx: ctx,cancel: cancel,stopClose: stopClose,@@ -400,12 +399,10 @@ func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containe}, size, nil}-func (s *Service) validateVolumeHelperSupportInternal(ctx context.Context, volumeName string) error {- dockerClient, err := s.deps.Docker.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to connect to Docker: %w", err)- }- result, err := dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{})+func (s *Service) validateVolumeHelperSupport(ctx context.Context, volumeName string) error {+ result, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{})+ })if err != nil {return fmt.Errorf("failed to inspect volume: %w", err)}@@ -415,17 +412,8 @@ func (s *Service) validateVolumeHelperSupportInternal(ctx context.Context, volumreturn nil}-func (s *Service) validateVolumeWorkspacePathInternal(ctx context.Context, containerID, relativePath string, allowMissing bool) error {- allowMissingValue := kit.Ternary(allowMissing, "1", "0")- _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceValidatePathScriptInternal, "sh", relativePath, allowMissingValue})- if err != nil {- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "validate volume workspace path")- }- return nil-}-const (- volumeWorkspaceValidatePathScriptInternal = `set -e+ volumeWorkspaceValidatePathScript = `set -erel="$1"allow_missing="$2"cur=/volume@@ -450,7 +438,7 @@ while :; doesacdone`- volumeWorkspaceBackupCreateScriptInternal = `set -e+ volumeWorkspaceBackupCreateScript = `set -erel="$1"archive="$2"cur=/volume@@ -484,18 +472,13 @@ tar -cf "$archive" "./$entry"printf 'present\0'`)-func (- s *Service,-) UpdateVolumeWorkspace(+func (s *Service) UpdateVolumeWorkspace(ctx context.Context,volumeName string,manifest volume.WorkspaceUpdateManifest,uploads map[int][]byte,user usertypes.Actor,-) (- *workspace.Workspace,- error,-) {+) (*workspace.Workspace, error) {totalStartedAt := time.Now()defer func() {slog.DebugContext(ctx, "volume workspace update completed", "volume", volumeName, "fileChangeCount", len(manifest.FileChanges), "totalDuration", time.Since(totalStartedAt))@@ -507,7 +490,7 @@ func (uploadReferences := make([]workspacepkg.UploadReference, 0, len(manifest.FileChanges))for _, change := range manifest.FileChanges {uploadReferences = append(uploadReferences, workspacepkg.UploadReference{Operation: change.Operation, UploadIndex: change.UploadIndex, BaselineIndex: change.BaselineIndex})- if err := validateVolumeWorkspaceFileChangeInternal(change); err != nil {+ if err := validateVolumeWorkspaceFileChange(change); err != nil {return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, err)}}@@ -515,8 +498,8 @@ func (return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, err)}inspectionStartedAt := time.Now()- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {- return nil, classifyVolumeWorkspaceHelperSupportErrorInternal(err)+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {+ return nil, classifyVolumeWorkspaceHelperSupportError(err)}slog.DebugContext(ctx, "volume workspace inspection completed", "volume", volumeName, "duration", time.Since(inspectionStartedAt))defer s.deps.Locks.Lock(volumeName)()@@ -525,7 +508,7 @@ func (return change.Operation == volume.FileOpRestoreFile})helperStartedAt := time.Now()- containerID, cleanup, err := s.createVolumeWorkspaceMutationContainerInternal(ctx, volumeName, needsBackups)+ containerID, cleanup, err := s.createVolumeWorkspaceMutationContainer(ctx, volumeName, needsBackups)if err != nil {return nil, err}@@ -536,7 +519,7 @@ func (slog.DebugContext(ctx, "volume workspace mutation helper acquired", "volume", volumeName, "containerId", containerID, "dedicated", needsBackups, "duration", time.Since(helperStartedAt))revisionScanStartedAt := time.Now()- current, err := s.readVolumeWorkspaceFromContainerInternal(ctx, containerID)+ current, err := s.readVolumeWorkspaceFromContainer(ctx, containerID)if err != nil {return nil, err}@@ -556,8 +539,8 @@ func (revisionScanStartedAt,),)- if validateVolumeWorkspaceRevisionErr := validateVolumeWorkspaceRevisionInternal(manifest.FileTreeRevision, current.FileTreeRevision); validateVolumeWorkspaceRevisionErr != nil {- return nil, validateVolumeWorkspaceRevisionErr+ if strings.TrimSpace(manifest.FileTreeRevision) != current.FileTreeRevision {+ return nil, common.Classify(common.ErrVolumeWorkspaceConflict, errors.New("volume workspace changed; refresh it and try again"))}stagingStartedAt := time.Now()@@ -565,31 +548,31 @@ func (if err != nil {return nil, err}- identity := s.resolveVolumeWorkspaceWriteIdentityInternal(ctx, dockerClient, containerID, volumeName)- stagedFiles, err := s.stageVolumeWorkspaceChangesInternal(ctx, dockerClient, containerID, manifest.FileChanges, uploads, identity)+ identity := s.resolveVolumeWorkspaceWriteIdentity(ctx, containerID, volumeName)+ stagedFiles, err := s.stageVolumeWorkspaceChanges(ctx, dockerClient, containerID, manifest.FileChanges, uploads, identity)if err != nil {return nil, err}slog.DebugContext(ctx, "volume workspace staging completed", "volume", volumeName, "stagedFileCount", len(stagedFiles), "duration", time.Since(stagingStartedAt))backupStartedAt := time.Now()- scope, err := volumeWorkspaceBackupScopeInternal(manifest.FileChanges)+ scope, err := volumeWorkspaceBackupScope(manifest.FileChanges)if err != nil {return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, err)}for _, relativePath := range scope {- if validateVolumeWorkspacePathErr := s.validateVolumeWorkspacePathInternal(ctx, containerID, relativePath, true); validateVolumeWorkspacePathErr != nil {- return nil, validateVolumeWorkspacePathErr+ if _, stderr, validateErr := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceValidatePathScript, "sh", relativePath, "1"}); validateErr != nil {+ return nil, classifyVolumeWorkspaceExecError(validateErr, stderr, "validate volume workspace path")}}- backup, err := s.backupVolumeWorkspaceScopeInternal(ctx, containerID, scope)+ backup, err := s.backupVolumeWorkspaceScope(ctx, containerID, scope)if err != nil {return nil, err}slog.DebugContext(ctx, "volume workspace backup completed", "volume", volumeName, "scopeCount", len(scope), "duration", time.Since(backupStartedAt))applyStartedAt := time.Now()- if applyVolumeWorkspaceChangesErr := s.applyVolumeWorkspaceChangesInternal(+ if applyVolumeWorkspaceChangesErr := s.applyVolumeWorkspaceChanges(ctx,dockerClient,containerID,@@ -597,14 +580,30 @@ func (manifest.FileChanges,stagedFiles,identity,- func() error { return s.restoreVolumeWorkspaceScopeInternal(ctx, containerID, backup) },+ func() error {+ args := append([]string{"sh", "-c", `set -e+for rel do rm -rf -- "/volume/$rel"; done`, "sh"}, volumeWorkspaceRollbackPaths(backup)...)+ if _, stderr, execErr := s.deps.Exec(ctx, containerID, "", args); execErr != nil {+ return classifyVolumeWorkspaceExecError(execErr, stderr, "remove failed volume workspace changes")+ }+ for _, archive := range backup.archives {+ containerParent := path.Join("/volume", path.Dir(archive.relativePath))+ if _, stderr, execErr := s.deps.Exec(ctx, containerID, "", []string{"mkdir", "-p", containerParent}); execErr != nil {+ return classifyVolumeWorkspaceExecError(execErr, stderr, "recreate volume workspace backup parent")+ }+ if _, stderr, execErr := s.deps.Exec(ctx, containerID, "", []string{"tar", "-xf", archive.archivePath, "-C", containerParent}); execErr != nil {+ return classifyVolumeWorkspaceExecError(execErr, stderr, "restore volume workspace backup")+ }+ }+ return nil+ },); applyVolumeWorkspaceChangesErr != nil {return nil, applyVolumeWorkspaceChangesErr}slog.DebugContext(ctx, "volume workspace changes applied", "volume", volumeName, "fileChangeCount", len(manifest.FileChanges), "duration", time.Since(applyStartedAt))finalScanStartedAt := time.Now()- tree, err := s.readVolumeWorkspaceFromContainerInternal(ctx, containerID)+ tree, err := s.readVolumeWorkspaceFromContainer(ctx, containerID)if err != nil {return nil, err}@@ -634,52 +633,56 @@ func (return tree, nil}-// volumeWorkspaceWriteIdentityInternal is the identity workspace file mutations run as,+// volumeWorkspaceWriteIdentity is the identity workspace file mutations run as,// so files land owned by the user the consuming container runs with. The zero value means root.-type volumeWorkspaceWriteIdentityInternal struct {+type volumeWorkspaceWriteIdentity struct {execUser stringuid intgid int}-func volumeWorkspaceWriteIdentityFromConfigUserInternal(configUser string) volumeWorkspaceWriteIdentityInternal {+func volumeWorkspaceWriteIdentityFromConfigUser(configUser string) volumeWorkspaceWriteIdentity {trimmed := strings.TrimSpace(configUser)if trimmed == "" {- return volumeWorkspaceWriteIdentityInternal{}+ return volumeWorkspaceWriteIdentity{}}// Named users cannot be resolved against the helper image, and uid 0 is already the default.uidPart, gidPart, hasGID := strings.Cut(trimmed, ":")uid, err := strconv.ParseUint(uidPart, 10, 32)if err != nil || uid == 0 {- return volumeWorkspaceWriteIdentityInternal{}+ return volumeWorkspaceWriteIdentity{}}var gid uint64if hasGID {gid, err = strconv.ParseUint(gidPart, 10, 32)if err != nil {- return volumeWorkspaceWriteIdentityInternal{}+ return volumeWorkspaceWriteIdentity{}}}- return volumeWorkspaceWriteIdentityInternal{execUser: trimmed, uid: int(uid), gid: int(gid)}+ return volumeWorkspaceWriteIdentity{execUser: trimmed, uid: int(uid), gid: int(gid)}}-func (s *Service) resolveVolumeWorkspaceWriteIdentityInternal(ctx context.Context, dockerClient *client.Client, containerID, volumeName string) volumeWorkspaceWriteIdentityInternal {- consumerIDs, err := docker.GetContainersUsingVolume(ctx, dockerClient, volumeName)+func (s *Service) resolveVolumeWorkspaceWriteIdentity(ctx context.Context, containerID, volumeName string) volumeWorkspaceWriteIdentity {+ consumerIDs, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]string, error) {+ return docker.GetContainersUsingVolume(ctx, dockerClient, volumeName)+ })if err != nil {slog.WarnContext(ctx, "could not list containers for volume workspace write identity", "volume", volumeName, "error", err.Error())}- identity := volumeWorkspaceWriteIdentityInternal{}+ identity := volumeWorkspaceWriteIdentity{}identitySource := ""for _, consumerID := range consumerIDs {- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, consumerID, client.ContainerInspectOptions{})+ inspect, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, consumerID, client.ContainerInspectOptions{})+ })if inspectErr != nil || inspect.Container.Config == nil {// An uninspectable consumer could declare a conflicting user, so the identity is// indeterminate; defer to the volume root's owner.slog.WarnContext(ctx, "could not inspect volume consumer; using volume root owner for workspace writes", "volume", volumeName, "containerId", consumerID)- identity = volumeWorkspaceWriteIdentityInternal{}+ identity = volumeWorkspaceWriteIdentity{}break}- parsed := volumeWorkspaceWriteIdentityFromConfigUserInternal(inspect.Container.Config.User)+ parsed := volumeWorkspaceWriteIdentityFromConfigUser(inspect.Container.Config.User)if parsed.execUser == "" {continue}@@ -700,7 +703,7 @@ func (s *Service) resolveVolumeWorkspaceWriteIdentityInternal(ctx context.Contex"conflictingIdentity",parsed.execUser,)- identity = volumeWorkspaceWriteIdentityInternal{}+ identity = volumeWorkspaceWriteIdentity{}break}}@@ -712,45 +715,40 @@ func (s *Service) resolveVolumeWorkspaceWriteIdentityInternal(ctx context.Contex// volume root is the next best signal for the runtime identity.stdout, _, err := s.deps.Exec(ctx, containerID, "", []string{"stat", "-c", "%u:%g", "/volume"})if err != nil {- return volumeWorkspaceWriteIdentityInternal{}+ return volumeWorkspaceWriteIdentity{}}- identity = volumeWorkspaceWriteIdentityFromConfigUserInternal(stdout)+ identity = volumeWorkspaceWriteIdentityFromConfigUser(stdout)if identity.execUser != "" {slog.InfoContext(ctx, "volume workspace writes run as volume root owner", "volume", volumeName, "identity", identity.execUser)}return identity}-type volumeWorkspaceStagedFileInternal struct {+type volumeWorkspaceStagedFile struct {path stringsize int64}-type volumeWorkspaceStagedContentInternal struct {+type volumeWorkspaceStagedContent struct {name stringcontent io.ReadClosersize int64}-func (- s *Service,-) stageVolumeWorkspaceChangesInternal(+func (s *Service) stageVolumeWorkspaceChanges(ctx context.Context,dockerClient *client.Client,containerID string,changes []volume.WorkspaceFileChange,uploads map[int][]byte,- identity volumeWorkspaceWriteIdentityInternal,-) (- map[int]volumeWorkspaceStagedFileInternal,- error,-) {+ identity volumeWorkspaceWriteIdentity,+) (map[int]volumeWorkspaceStagedFile, error) {if _, _, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", "rm -rf -- /tmp/arcane-workspace && mkdir -p -- /tmp/arcane-workspace"}); err != nil {return nil, fmt.Errorf("prepare volume workspace staging directory: %w", err)}- stagedFiles := make(map[int]volumeWorkspaceStagedFileInternal)- stagedContents := make([]volumeWorkspaceStagedContentInternal, 0, len(changes))+ stagedFiles := make(map[int]volumeWorkspaceStagedFile)+ stagedContents := make([]volumeWorkspaceStagedContent, 0, len(changes))for index, change := range changes {if change.UploadIndex == nil {continue@@ -759,8 +757,8 @@ func (content := uploads[*change.UploadIndex]reader := io.NopCloser(bytes.NewReader(content))size := int64(len(content))- stagedFiles[index] = volumeWorkspaceStagedFileInternal{path: stagedPath, size: size}- stagedContents = append(stagedContents, volumeWorkspaceStagedContentInternal{+ stagedFiles[index] = volumeWorkspaceStagedFile{path: stagedPath, size: size}+ stagedContents = append(stagedContents, volumeWorkspaceStagedContent{name: path.Base(stagedPath),content: reader,size: size,@@ -769,23 +767,23 @@ func (if len(stagedContents) == 0 {return stagedFiles, nil}- if err := s.copyVolumeWorkspaceFilesToContainerInternal(ctx, dockerClient, containerID, stagedContents, identity); err != nil {+ if err := s.copyVolumeWorkspaceFilesToContainer(ctx, dockerClient, containerID, stagedContents, identity); err != nil {return nil, err}return stagedFiles, nil}-func (s *Service) applyVolumeWorkspaceChangesInternal(+func (s *Service) applyVolumeWorkspaceChanges(ctx context.Context,dockerClient *client.Client,containerID string,volumeName string,changes []volume.WorkspaceFileChange,- stagedFiles map[int]volumeWorkspaceStagedFileInternal,- identity volumeWorkspaceWriteIdentityInternal,+ stagedFiles map[int]volumeWorkspaceStagedFile,+ identity volumeWorkspaceWriteIdentity,rollback func() error,) error {- rollbackFailureInternal := func(applyErr error) error {+ rollbackFailure := func(applyErr error) error {if rollbackErr := rollback(); rollbackErr != nil {return errors.Join(applyErr, fmt.Errorf("failed to roll back volume workspace: %w", rollbackErr))}@@ -795,8 +793,19 @@ func (s *Service) applyVolumeWorkspaceChangesInternal(for index := 0; index < len(changes); {if changes[index].Operation == volume.FileOpRestoreFile {rel, _ := kit.NormalizeRelativePath(changes[index].RelativePath)- if err := s.restoreVolumeWorkspaceFileInternal(ctx, containerID, volumeName, changes[index].BackupID, rel); err != nil {- return rollbackFailureInternal(err)+ if _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceValidatePathScript, "sh", rel, "1"}); err != nil {+ return rollbackFailure(classifyVolumeWorkspaceExecError(err, stderr, "validate volume workspace path"))+ }+ filename, cleaned, err := s.deps.ArchiveFileTarget(ctx, volumeName, changes[index].BackupID, rel)+ if err != nil {+ return rollbackFailure(err)+ }+ stderr, err := s.deps.RestoreArchiveMembers(ctx, containerID, filename, []string{cleaned})+ if err != nil {+ return rollbackFailure(classifyVolumeWorkspaceExecError(err, stderr, "restore volume workspace file"))+ }+ if strings.TrimSpace(stderr) != "" {+ slog.DebugContext(ctx, "volume workspace change stderr", "operation", volume.FileOpRestoreFile, "path", rel, "stderr", strings.TrimSpace(stderr))}index++continue@@ -806,26 +815,26 @@ func (s *Service) applyVolumeWorkspaceChangesInternal(for end < len(changes) && changes[end].Operation != volume.FileOpRestoreFile {end++}- if err := s.executeVolumeWorkspaceACFSBatchInternal(ctx, dockerClient, containerID, changes[index:end], stagedFiles, index, identity); err != nil {- return rollbackFailureInternal(err)+ if err := s.executeVolumeWorkspaceACFSBatch(ctx, dockerClient, containerID, changes[index:end], stagedFiles, index, identity); err != nil {+ return rollbackFailure(err)}index = end}return nil}-func (s *Service) executeVolumeWorkspaceACFSBatchInternal(+func (s *Service) executeVolumeWorkspaceACFSBatch(ctx context.Context,dockerClient *client.Client,containerID string,changes []volume.WorkspaceFileChange,- stagedFiles map[int]volumeWorkspaceStagedFileInternal,+ stagedFiles map[int]volumeWorkspaceStagedFile,startIndex int,- identity volumeWorkspaceWriteIdentityInternal,+ identity volumeWorkspaceWriteIdentity,) error {applyChanges := make([]types.ApplyChange, 0, len(changes))for offset, change := range changes {- applyChange, err := mapVolumeWorkspaceChangeToACFSInternal(change, stagedFiles[startIndex+offset])+ applyChange, err := mapVolumeWorkspaceChangeToACFS(change, stagedFiles[startIndex+offset])if err != nil {return common.Classify(common.ErrVolumeWorkspaceBadRequest, err)}@@ -840,7 +849,7 @@ func (s *Service) executeVolumeWorkspaceACFSBatchInternal(return fmt.Errorf("encode volume workspace apply manifest: %w", err)}manifestName := fmt.Sprintf("manifest-%d.json", startIndex)- if copyVolumeWorkspaceFilesToContainerErr := s.copyVolumeWorkspaceFilesToContainerInternal(ctx, dockerClient, containerID, []volumeWorkspaceStagedContentInternal{{+ if copyVolumeWorkspaceFilesToContainerErr := s.copyVolumeWorkspaceFilesToContainer(ctx, dockerClient, containerID, []volumeWorkspaceStagedContent{{name: manifestName,content: io.NopCloser(bytes.NewReader(manifestContent)),size: int64(len(manifestContent)),@@ -852,7 +861,7 @@ func (s *Service) executeVolumeWorkspaceACFSBatchInternal("acfs", "apply", "--root", "/volume", "--staging", "/tmp/arcane-workspace", "--manifest", manifestName,})if err != nil {- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "apply volume workspace changes")+ return classifyVolumeWorkspaceExecError(err, stderr, "apply volume workspace changes")}var response types.ApplyResponseif unmarshalErr := json.Unmarshal([]byte(stdout), &response); unmarshalErr != nil {@@ -864,59 +873,61 @@ func (s *Service) executeVolumeWorkspaceACFSBatchInternal(return nil}-func mapVolumeWorkspaceChangeToACFSInternal(change volume.WorkspaceFileChange, staged volumeWorkspaceStagedFileInternal) (types.ApplyChange, error) {+func mapVolumeWorkspaceChangeToACFS(change volume.WorkspaceFileChange, staged volumeWorkspaceStagedFile) (types.ApplyChange, error) {relativePath, err := kit.NormalizeRelativePath(change.RelativePath)if err != nil {return types.ApplyChange{}, err}- result := types.ApplyChange{Path: "/" + relativePath, Recursive: change.Recursive}switch change.Operation {- case volume.FileOpCreateFile:- result.Operation = types.ApplyCreateFile- result.StagedName = path.Base(staged.path)- result.Size = staged.size- case volume.FileOpUpdateFile:- result.Operation = types.ApplyUpdateFile- result.StagedName = path.Base(staged.path)- result.Size = staged.size+ case volume.FileOpCreateFile, volume.FileOpUpdateFile:+ return types.ApplyChange{+ Operation: kit.Ternary(change.Operation == volume.FileOpCreateFile, types.ApplyCreateFile, types.ApplyUpdateFile),+ Path: "/" + relativePath,+ StagedName: path.Base(staged.path),+ Size: staged.size,+ Recursive: change.Recursive,+ }, nilcase volume.FileOpCreateFolder:- result.Operation = types.ApplyCreateFolder- result.Recursive = false+ return types.ApplyChange{Operation: types.ApplyCreateFolder, Path: "/" + relativePath}, nilcase volume.FileOpRename:newName, nameErr := kit.ValidateFileName(change.NewName)if nameErr != nil {return types.ApplyChange{}, nameErr}- result.Operation = types.ApplyRename- result.TargetPath = "/" + path.Join(path.Dir(relativePath), newName)- result.Recursive = false+ return types.ApplyChange{+ Operation: types.ApplyRename,+ Path: "/" + relativePath,+ TargetPath: "/" + path.Join(path.Dir(relativePath), newName),+ }, nilcase volume.FileOpMove:parent := ""if strings.TrimSpace(change.NewParentPath) != "" {- parent, err = kit.NormalizeRelativePath(change.NewParentPath)- if err != nil {+ if parent, err = kit.NormalizeRelativePath(change.NewParentPath); err != nil {return types.ApplyChange{}, err}}- result.Operation = types.ApplyMove- result.TargetPath = "/" + path.Join(parent, path.Base(relativePath))- result.Recursive = false+ return types.ApplyChange{+ Operation: types.ApplyMove,+ Path: "/" + relativePath,+ TargetPath: "/" + path.Join(parent, path.Base(relativePath)),+ }, nilcase volume.FileOpDelete:- result.Operation = types.ApplyDelete+ return types.ApplyChange{+ Operation: types.ApplyDelete,+ Path: "/" + relativePath,+ Recursive: change.Recursive,+ }, nildefault:return types.ApplyChange{}, fmt.Errorf("unsupported volume workspace operation %q", change.Operation)}- return result, nil}-func (- s *Service,-) copyVolumeWorkspaceFilesToContainerInternal(+func (s *Service) copyVolumeWorkspaceFilesToContainer(ctx context.Context,dockerClient *client.Client,containerID string,- contents []volumeWorkspaceStagedContentInternal,- identity volumeWorkspaceWriteIdentityInternal,+ contents []volumeWorkspaceStagedContent,+ identity volumeWorkspaceWriteIdentity,) error {pipeReader, pipeWriter := io.Pipe()archiveDone := make(chan error, 1)@@ -952,7 +963,7 @@ func (return nil}-func (s *Service) createVolumeWorkspaceMutationContainerInternal(ctx context.Context, volumeName string, withBackups bool) (string, func(), error) {+func (s *Service) createVolumeWorkspaceMutationContainer(ctx context.Context, volumeName string, withBackups bool) (string, func(), error) {if !withBackups {return s.deps.AcquireHelper(ctx, volumeName)}@@ -960,7 +971,7 @@ func (s *Service) createVolumeWorkspaceMutationContainerInternal(ctx context.Conif err != nil {return "", nil, err}- helperImage, err := s.deps.HelperImage(ctx, dockerClient)+ helperImage, err := s.deps.HelperImage(ctx)if err != nil {return "", nil, err}@@ -970,52 +981,58 @@ func (s *Service) createVolumeWorkspaceMutationContainerInternal(ctx context.Con}config := &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()}hostConfig := volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume"}, []mount.Mount{backupMount})- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: config, HostConfig: hostConfig})+ resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: config, HostConfig: hostConfig})+ })if err != nil {return "", nil, fmt.Errorf("create volume workspace helper: %w", err)}- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {+ if _, containerStartErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {startErr := fmt.Errorf("start volume workspace helper: %w", containerStartErr)- cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)- defer cancel()- if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {+ if _, cleanupErr := s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ }); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {return "", nil, errors.Join(startErr, fmt.Errorf("remove volume workspace helper after start failure: %w", cleanupErr))}return "", nil, startErr}return resp.ID, func() {- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ })}, nil}-type volumeWorkspaceBackupArchiveInternal struct {+type volumeWorkspaceBackupArchive struct {relativePath stringarchivePath string}-type volumeWorkspaceBackupInternal struct {- archives []volumeWorkspaceBackupArchiveInternal+type volumeWorkspaceBackup struct {+ archives []volumeWorkspaceBackupArchiveabsentEntries []string}-func (s *Service) backupVolumeWorkspaceScopeInternal(ctx context.Context, containerID string, scope []string) (*volumeWorkspaceBackupInternal, error) {+func (s *Service) backupVolumeWorkspaceScope(ctx context.Context, containerID string, scope []string) (*volumeWorkspaceBackup, error) {if _, _, err := s.deps.Exec(ctx, containerID, "", []string{"mkdir", "-p", "/tmp/arcane-workspace"}); err != nil {return nil, fmt.Errorf("prepare volume workspace backup directory: %w", err)}- backup := &volumeWorkspaceBackupInternal{- archives: make([]volumeWorkspaceBackupArchiveInternal, 0, len(scope)),+ backup := &volumeWorkspaceBackup{+ archives: make([]volumeWorkspaceBackupArchive, 0, len(scope)),absentEntries: make([]string, 0, len(scope)),}for index, relativePath := range scope {archivePath := fmt.Sprintf("/tmp/arcane-workspace/backup-%d.tar", index)- stdout, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceBackupCreateScriptInternal, "sh", relativePath, archivePath})+ stdout, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceBackupCreateScript, "sh", relativePath, archivePath})if err != nil {- return nil, classifyVolumeWorkspaceExecErrorInternal(err, stderr, "back up volume workspace path")+ return nil, classifyVolumeWorkspaceExecError(err, stderr, "back up volume workspace path")}fields := strings.Split(stdout, "\x00")switch {case len(fields) >= 1 && fields[0] == "present":- backup.archives = append(backup.archives, volumeWorkspaceBackupArchiveInternal{+ backup.archives = append(backup.archives, volumeWorkspaceBackupArchive{relativePath: relativePath,archivePath: archivePath,})@@ -1025,11 +1042,11 @@ func (s *Service) backupVolumeWorkspaceScopeInternal(ctx context.Context, contaireturn nil, errors.New("invalid volume workspace backup path response")}}- backup.absentEntries = normalizeVolumeWorkspaceScopeInternal(backup.absentEntries)+ backup.absentEntries = normalizeVolumeWorkspaceScope(backup.absentEntries)return backup, nil}-func volumeWorkspaceRollbackPathsInternal(backup *volumeWorkspaceBackupInternal) []string {+func volumeWorkspaceRollbackPaths(backup *volumeWorkspaceBackup) []string {removePaths := slices.Clone(backup.absentEntries)for _, archive := range backup.archives {removePaths = append(removePaths, archive.relativePath)@@ -1045,49 +1062,7 @@ func volumeWorkspaceRollbackPathsInternal(backup *volumeWorkspaceBackupInternal)return removePaths}-func (s *Service) restoreVolumeWorkspaceScopeInternal(ctx context.Context, containerID string, backup *volumeWorkspaceBackupInternal) error {- removePaths := volumeWorkspaceRollbackPathsInternal(backup)- args := append([]string{"sh", "-c", `set -e-for rel do rm -rf -- "/volume/$rel"; done`, "sh"}, removePaths...)- if _, stderr, err := s.deps.Exec(ctx, containerID, "", args); err != nil {- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "remove failed volume workspace changes")- }-- for _, archive := range backup.archives {- parent := path.Dir(archive.relativePath)- containerParent := "/volume"- if parent != "." {- containerParent = path.Join(containerParent, parent)- }- if _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"mkdir", "-p", containerParent}); err != nil {- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "recreate volume workspace backup parent")- }- if _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"tar", "-xf", archive.archivePath, "-C", containerParent}); err != nil {- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "restore volume workspace backup")- }- }- return nil-}--func (s *Service) restoreVolumeWorkspaceFileInternal(ctx context.Context, containerID, volumeName, backupID, rel string) error {- if err := s.validateVolumeWorkspacePathInternal(ctx, containerID, rel, true); err != nil {- return err- }- filename, cleaned, err := s.deps.ArchiveFileTarget(ctx, volumeName, backupID, rel)- if err != nil {- return err- }- stderr, err := s.deps.RestoreArchiveMembers(ctx, containerID, filename, []string{cleaned})- if err != nil {- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "restore volume workspace file")- }- if strings.TrimSpace(stderr) != "" {- slog.DebugContext(ctx, "volume workspace change stderr", "operation", volume.FileOpRestoreFile, "path", rel, "stderr", strings.TrimSpace(stderr))- }- return nil-}--func normalizeVolumeWorkspaceScopeInternal(paths []string) []string {+func normalizeVolumeWorkspaceScope(paths []string) []string {normalized := slices.Clone(paths)slices.Sort(normalized)normalized = slices.Compact(normalized)@@ -1104,7 +1079,7 @@ func normalizeVolumeWorkspaceScopeInternal(paths []string) []string {return result}-func volumeWorkspaceBackupScopeInternal(changes []volume.WorkspaceFileChange) ([]string, error) {+func volumeWorkspaceBackupScope(changes []volume.WorkspaceFileChange) ([]string, error) {paths := make([]string, 0, len(changes)*2)for _, change := range changes {rel, err := kit.NormalizeRelativePath(change.RelativePath)@@ -1130,10 +1105,10 @@ func volumeWorkspaceBackupScopeInternal(changes []volume.WorkspaceFileChange) ([paths = append(paths, path.Join(parent, path.Base(rel)))}}- return normalizeVolumeWorkspaceScopeInternal(paths), nil+ return normalizeVolumeWorkspaceScope(paths), nil}-func validateVolumeWorkspaceFileChangeInternal(change volume.WorkspaceFileChange) error {+func validateVolumeWorkspaceFileChange(change volume.WorkspaceFileChange) error {if _, err := kit.NormalizeRelativePath(change.RelativePath); err != nil {return fmt.Errorf("invalid volume workspace path: %w", err)}@@ -1176,14 +1151,7 @@ func validateVolumeWorkspaceFileChangeInternal(change volume.WorkspaceFileChangereturn nil}-func validateVolumeWorkspaceRevisionInternal(expected, current string) error {- if strings.TrimSpace(expected) != current {- return common.Classify(common.ErrVolumeWorkspaceConflict, errors.New("volume workspace changed; refresh it and try again"))- }- return nil-}--func classifyVolumeWorkspaceHelperSupportErrorInternal(err error) error {+func classifyVolumeWorkspaceHelperSupportError(err error) error {switch {case errdefs.IsNotFound(err):return common.Classify(common.ErrVolumeWorkspaceNotFound, err)@@ -1194,7 +1162,7 @@ func classifyVolumeWorkspaceHelperSupportErrorInternal(err error) error {}}-func classifyVolumeWorkspaceExecErrorInternal(err error, stderr, fallbackContext string) error {+func classifyVolumeWorkspaceExecError(err error, stderr, fallbackContext string) error {var response types.ErrorResponseif decodeErr := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &response); decodeErr == nil && response.Version == types.ProtocolVersion {switch response.Code {@@ -1240,35 +1208,34 @@ func classifyVolumeWorkspaceExecErrorInternal(err error, stderr, fallbackContext}}-func volumeWorkspaceFileContentResponseInternal(relativePath, kind string, size int64, content []byte, maxFileSizeBytes int64) (*workspace.FileContent, error) {- response := &workspace.FileContent{- Path: "/" + relativePath,- RelativePath: relativePath,- Name: path.Base(relativePath),- MimeType: http.DetectContentType(content),- Size: size,- }- if kind == "special" {- response.ReadOnlyReason = workspace.FileReadOnlySpecial- return response, nil- }- if kind != "regular" {+func volumeWorkspaceFileContentResponse(relativePath, kind string, size int64, content []byte, maxFileSizeBytes int64) (*workspace.FileContent, error) {+ if kind != "special" && kind != "regular" {return nil, errors.New("invalid volume workspace file kind")}- if size > maxFileSizeBytes {- response.ReadOnlyReason = workspace.FileReadOnlyTooLarge- return response, nil- }- if !workspacepkg.IsTextContent(content) {- response.ReadOnlyReason = workspace.FileReadOnlyBinary- return response, nil- }- response.Editable = true- response.Content = string(content)- return response, nil+ readOnlyReason := func() string {+ switch {+ case kind == "special":+ return workspace.FileReadOnlySpecial+ case size > maxFileSizeBytes:+ return workspace.FileReadOnlyTooLarge+ case !workspacepkg.IsTextContent(content):+ return workspace.FileReadOnlyBinary+ }+ return ""+ }()+ return &workspace.FileContent{+ Path: "/" + relativePath,+ RelativePath: relativePath,+ Name: path.Base(relativePath),+ MimeType: http.DetectContentType(content),+ Size: size,+ ReadOnlyReason: readOnlyReason,+ Editable: readOnlyReason == "",+ Content: kit.Ternary(readOnlyReason == "", string(content), ""),+ }, nil}-func volumeWorkspaceEntryFromACFSInternal(entry types.Entry, maxFileSizeBytes int64) (workspace.FileEntry, string) {+func volumeWorkspaceEntryFromACFS(entry types.Entry, maxFileSizeBytes int64) (workspace.FileEntry, string) {fileEntry := acfsutils.FileEntry(entry)fileEntry.RelativePath = strings.TrimPrefix(entry.Path, "/")switch {@@ -1292,7 +1259,7 @@ func volumeWorkspaceEntryFromACFSInternal(entry types.Entry, maxFileSizeBytes in}}-func decodeVolumeWorkspaceWalkInternal(source io.Reader, maxEntries int, maxFileSizeBytes int64) (*workspace.Workspace, error) {+func decodeVolumeWorkspaceWalk(source io.Reader, maxEntries int, maxFileSizeBytes int64) (*workspace.Workspace, error) {decoder := jsontext.NewDecoder(source)files := make([]workspace.FileEntry, 0, min(maxEntries, 256))classifications := make(map[string]string, min(maxEntries, 256))@@ -1327,7 +1294,7 @@ func decodeVolumeWorkspaceWalkInternal(source io.Reader, maxEntries int, maxFileif len(files) >= maxEntries {return nil, errors.New("acfs walk exceeded the requested entry limit")}- fileEntry, classification := volumeWorkspaceEntryFromACFSInternal(*record.Entry, maxFileSizeBytes)+ fileEntry, classification := volumeWorkspaceEntryFromACFS(*record.Entry, maxFileSizeBytes)classifications[fileEntry.RelativePath] = classificationfiles = append(files, fileEntry)}diff --git a/backend/internal/volume/children/workspace/service_test.go b/backend/internal/volume/children/workspace/service_test.goindex d23b048a3..ebebe0b9b 100644--- a/backend/internal/volume/children/workspace/service_test.go+++ b/backend/internal/volume/children/workspace/service_test.go@@ -31,7 +31,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper")-func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {+func newVolumeWorkspaceTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {t.Helper()dockerClient, err := client.New(client.WithHost(server.URL),@@ -43,7 +43,7 @@ func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.Sreturn dockerClient}-func TestValidateVolumeHelperSupportInternal(t *testing.T) {+func TestValidateVolumeHelperSupport(t *testing.T) {var requests []stringvar options map[string]stringserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -59,18 +59,18 @@ func TestValidateVolumeHelperSupportInternal(t *testing.T) {}))t.Cleanup(server.Close)- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClient(t, server))service := NewService(Dependencies{Docker: dockerService})- require.NoError(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))+ require.NoError(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))require.Equal(t, []string{"GET /v1.41/volumes/workspace-volume"}, requests)options = map[string]string{"type": "nfs"}- require.NoError(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))+ require.NoError(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))options = map[string]string{"type": "none"}- require.Error(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))+ require.Error(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))options = map[string]string{"o": "bind,rw"}- require.Error(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))+ require.Error(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))}func TestUpdateVolumeWorkspaceValidationFailureReturnsNoWorkspace(t *testing.T) {@@ -86,33 +86,33 @@ func TestUpdateVolumeWorkspaceValidationFailureReturnsNoWorkspace(t *testing.T)require.ErrorIs(t, err, common.ErrVolumeWorkspaceBadRequest)}-func TestVolumeWorkspaceFileContentResponseInternal(t *testing.T) {+func TestVolumeWorkspaceFileContentResponse(t *testing.T) {const maxFileSizeBytes int64 = 10 * 1024 * 1024text := []byte("hello")- below, err := volumeWorkspaceFileContentResponseInternal("notes.txt", "regular", maxFileSizeBytes-1, text, maxFileSizeBytes)+ below, err := volumeWorkspaceFileContentResponse("notes.txt", "regular", maxFileSizeBytes-1, text, maxFileSizeBytes)require.NoError(t, err)require.True(t, below.Editable)require.Equal(t, "hello", below.Content)- atLimit, err := volumeWorkspaceFileContentResponseInternal("notes.txt", "regular", maxFileSizeBytes, text, maxFileSizeBytes)+ atLimit, err := volumeWorkspaceFileContentResponse("notes.txt", "regular", maxFileSizeBytes, text, maxFileSizeBytes)require.NoError(t, err)require.True(t, atLimit.Editable)- above, err := volumeWorkspaceFileContentResponseInternal("notes.txt", "regular", maxFileSizeBytes+1, text, maxFileSizeBytes)+ above, err := volumeWorkspaceFileContentResponse("notes.txt", "regular", maxFileSizeBytes+1, text, maxFileSizeBytes)require.NoError(t, err)require.False(t, above.Editable)require.Equal(t, workspace.FileReadOnlyTooLarge, above.ReadOnlyReason)- binaryContent, err := volumeWorkspaceFileContentResponseInternal("data.bin", "regular", 2, []byte{0xff, 0x00}, maxFileSizeBytes)+ binaryContent, err := volumeWorkspaceFileContentResponse("data.bin", "regular", 2, []byte{0xff, 0x00}, maxFileSizeBytes)require.NoError(t, err)require.Equal(t, workspace.FileReadOnlyBinary, binaryContent.ReadOnlyReason)- special, err := volumeWorkspaceFileContentResponseInternal("pipe", "special", 0, nil, maxFileSizeBytes)+ special, err := volumeWorkspaceFileContentResponse("pipe", "special", 0, nil, maxFileSizeBytes)require.NoError(t, err)require.Equal(t, workspace.FileReadOnlySpecial, special.ReadOnlyReason)}-func TestValidateVolumeWorkspaceFileChangeInternalOperationsAndMultipartMapping(t *testing.T) {+func TestValidateVolumeWorkspaceFileChangeOperationsAndMultipartMapping(t *testing.T) {first := 0second := 1tests := []volumetypes.WorkspaceFileChange{@@ -126,36 +126,36 @@ func TestValidateVolumeWorkspaceFileChangeInternalOperationsAndMultipartMapping({Operation: volumetypes.FileOpRestoreFile, RelativePath: "restored.txt", BackupID: "backup-id"},}for _, change := range tests {- require.NoError(t, validateVolumeWorkspaceFileChangeInternal(change), change.Operation)+ require.NoError(t, validateVolumeWorkspaceFileChange(change), change.Operation)}- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{Operation: volumetypes.FileOpUpdateFile, RelativePath: "missing.bin",}))- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{Operation: volumetypes.FileOpRestoreFile, RelativePath: "file.txt",}))- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{Operation: "unknown", RelativePath: "file.txt",}))- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{Operation: volumetypes.FileOpDelete, RelativePath: "../escape",}))}-func TestVolumeWorkspaceBackupScopeInternalCoversSourcesAndDestinations(t *testing.T) {+func TestVolumeWorkspaceBackupScopeCoversSourcesAndDestinations(t *testing.T) {changes := []volumetypes.WorkspaceFileChange{{Operation: volumetypes.FileOpRename, RelativePath: "docs/a.txt", NewName: "b.txt"},{Operation: volumetypes.FileOpMove, RelativePath: "cache", NewParentPath: "docs"},{Operation: volumetypes.FileOpCreateFile, RelativePath: "docs/a.txt/child", UploadIndex: new(0)},}- scope, err := volumeWorkspaceBackupScopeInternal(changes)+ scope, err := volumeWorkspaceBackupScope(changes)require.NoError(t, err)require.Equal(t, []string{"cache", "docs/a.txt", "docs/b.txt", "docs/cache"}, scope)}-func TestNormalizeVolumeWorkspaceScopeInternalCollapsesAbsentAncestors(t *testing.T) {- require.Equal(t, []string{"cache", "docs"}, normalizeVolumeWorkspaceScopeInternal([]string{+func TestNormalizeVolumeWorkspaceScopeCollapsesAbsentAncestors(t *testing.T) {+ require.Equal(t, []string{"cache", "docs"}, normalizeVolumeWorkspaceScope([]string{"docs/b.txt","docs","cache/item",@@ -165,21 +165,21 @@ func TestNormalizeVolumeWorkspaceScopeInternalCollapsesAbsentAncestors(t *testin}))}-func TestVolumeWorkspaceRollbackPathsInternalRemovesDeepestFirst(t *testing.T) {- backup := &volumeWorkspaceBackupInternal{- archives: []volumeWorkspaceBackupArchiveInternal{+func TestVolumeWorkspaceRollbackPathsRemovesDeepestFirst(t *testing.T) {+ backup := &volumeWorkspaceBackup{+ archives: []volumeWorkspaceBackupArchive{{relativePath: "docs/file.txt", archivePath: "/tmp/file.tar"},{relativePath: "cache", archivePath: "/tmp/cache.tar"},},absentEntries: []string{"docs/new/deep", "other"},}- require.Equal(t, []string{"docs/new/deep", "docs/file.txt", "other", "cache"}, volumeWorkspaceRollbackPathsInternal(backup))+ require.Equal(t, []string{"docs/new/deep", "docs/file.txt", "other", "cache"}, volumeWorkspaceRollbackPaths(backup))}func TestVolumeWorkspaceHelperScriptsUseSupportedTooling(t *testing.T) {helperScripts := []string{- volumeWorkspaceValidatePathScriptInternal,- volumeWorkspaceBackupCreateScriptInternal,+ volumeWorkspaceValidatePathScript,+ volumeWorkspaceBackupCreateScript,}scripts := strings.Join(helperScripts, "\n")for _, unsupported := range []string{@@ -201,9 +201,9 @@ func TestVolumeWorkspaceHelperScriptsUseSupportedTooling(t *testing.T) {} {require.NotContains(t, scripts, unsupported)}- require.Contains(t, volumeWorkspaceBackupCreateScriptInternal, `printf 'absent\0%s\0'`)- require.Contains(t, volumeWorkspaceBackupCreateScriptInternal, `cd "$parent"`)- require.NotContains(t, volumeWorkspaceBackupCreateScriptInternal, " -C ")+ require.Contains(t, volumeWorkspaceBackupCreateScript, `printf 'absent\0%s\0'`)+ require.Contains(t, volumeWorkspaceBackupCreateScript, `cd "$parent"`)+ require.NotContains(t, volumeWorkspaceBackupCreateScript, " -C ")if shellPath, err := exec.LookPath("sh"); err == nil {for index, script := range helperScripts {output, syntaxErr := exec.Command(shellPath, "-n", "-c", script).CombinedOutput()@@ -212,24 +212,19 @@ func TestVolumeWorkspaceHelperScriptsUseSupportedTooling(t *testing.T) {}}-func TestClassifyVolumeWorkspaceExecErrorInternal(t *testing.T) {+func TestClassifyVolumeWorkspaceExecError(t *testing.T) {base := errors.New("exit status 1")- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "ARCANE_SYMLINK", "execute"), common.ErrVolumeWorkspaceForbidden)- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "ARCANE_NOT_FOUND", "execute"), common.ErrVolumeWorkspaceNotFound)- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "ARCANE_COLLISION", "execute"), common.ErrVolumeWorkspaceBadRequest)- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "unexpected", "execute"), base)+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "ARCANE_SYMLINK", "execute"), common.ErrVolumeWorkspaceForbidden)+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "ARCANE_NOT_FOUND", "execute"), common.ErrVolumeWorkspaceNotFound)+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "ARCANE_COLLISION", "execute"), common.ErrVolumeWorkspaceBadRequest)+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "unexpected", "execute"), base)}-func TestClassifyVolumeWorkspaceHelperSupportErrorInternal(t *testing.T) {- require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportErrorInternal(errdefs.ErrNotFound), common.ErrVolumeWorkspaceNotFound)- require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportErrorInternal(errors.New("volume uses a custom mount configuration")), common.ErrVolumeWorkspaceBadRequest)+func TestClassifyVolumeWorkspaceHelperSupportError(t *testing.T) {+ require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportError(errdefs.ErrNotFound), common.ErrVolumeWorkspaceNotFound)+ require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportError(errors.New("volume uses a custom mount configuration")), common.ErrVolumeWorkspaceBadRequest)unexpected := errors.New("docker unavailable")- require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportErrorInternal(unexpected), unexpected)-}--func TestValidateVolumeWorkspaceRevisionInternal(t *testing.T) {- require.NoError(t, validateVolumeWorkspaceRevisionInternal(" revision ", "revision"))- require.ErrorIs(t, validateVolumeWorkspaceRevisionInternal("stale", "current"), common.ErrVolumeWorkspaceConflict)+ require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportError(unexpected), unexpected)}func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testing.T) {@@ -267,7 +262,7 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi}))t.Cleanup(server.Close)- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)service := NewService(Dependencies{Docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),Exec: func(_ context.Context, containerID, _ string, cmd []string) (string, string, error) {@@ -278,16 +273,16 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi})firstUpload := 0secondUpload := 1- firstStaged, err := service.stageVolumeWorkspaceChangesInternal(t.Context(), dockerClient, "helper", []volumetypes.WorkspaceFileChange{+ firstStaged, err := service.stageVolumeWorkspaceChanges(t.Context(), dockerClient, "helper", []volumetypes.WorkspaceFileChange{{UploadIndex: &firstUpload},{},{UploadIndex: &secondUpload},- }, map[int][]byte{0: []byte("alpha"), 1: {}}, volumeWorkspaceWriteIdentityInternal{})+ }, map[int][]byte{0: []byte("alpha"), 1: {}}, volumeWorkspaceWriteIdentity{})require.NoError(t, err)- require.Equal(t, volumeWorkspaceStagedFileInternal{path: "/tmp/arcane-workspace/change-0", size: 5}, firstStaged[0])- require.Equal(t, volumeWorkspaceStagedFileInternal{path: "/tmp/arcane-workspace/change-2", size: 0}, firstStaged[2])+ require.Equal(t, volumeWorkspaceStagedFile{path: "/tmp/arcane-workspace/change-0", size: 5}, firstStaged[0])+ require.Equal(t, volumeWorkspaceStagedFile{path: "/tmp/arcane-workspace/change-2", size: 0}, firstStaged[2])- secondStaged, err := service.stageVolumeWorkspaceChangesInternal(+ secondStaged, err := service.stageVolumeWorkspaceChanges(t.Context(),dockerClient,"helper",@@ -301,10 +296,10 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi"second",),},- volumeWorkspaceWriteIdentityInternal{},+ volumeWorkspaceWriteIdentity{},)require.NoError(t, err)- require.Equal(t, volumeWorkspaceStagedFileInternal{path: "/tmp/arcane-workspace/change-0", size: 6}, secondStaged[0])+ require.Equal(t, volumeWorkspaceStagedFile{path: "/tmp/arcane-workspace/change-0", size: 6}, secondStaged[0])require.Equal(t, [][]string{{"sh", "-c", "rm -rf -- /tmp/arcane-workspace && mkdir -p -- /tmp/arcane-workspace"},@@ -316,22 +311,22 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi}, copiedArchives)}-func TestVolumeWorkspaceWriteIdentityFromConfigUserInternal(t *testing.T) {- root := volumeWorkspaceWriteIdentityInternal{}+func TestVolumeWorkspaceWriteIdentityFromConfigUser(t *testing.T) {+ root := volumeWorkspaceWriteIdentity{}for _, configUser := range []string{"", "root", "0", "0:0", "0:1000", "node", "1000:node", "99999999999999999999"} {- require.Equalf(t, root, volumeWorkspaceWriteIdentityFromConfigUserInternal(configUser), "config user %q", configUser)+ require.Equalf(t, root, volumeWorkspaceWriteIdentityFromConfigUser(configUser), "config user %q", configUser)}- require.Equal(t, volumeWorkspaceWriteIdentityInternal{execUser: "1000", uid: 1000}, volumeWorkspaceWriteIdentityFromConfigUserInternal("1000"))- require.Equal(t, volumeWorkspaceWriteIdentityInternal{execUser: "1000:1000", uid: 1000, gid: 1000}, volumeWorkspaceWriteIdentityFromConfigUserInternal(" 1000:1000 "))- require.Equal(t, volumeWorkspaceWriteIdentityInternal{execUser: "1000:0", uid: 1000, gid: 0}, volumeWorkspaceWriteIdentityFromConfigUserInternal("1000:0"))+ require.Equal(t, volumeWorkspaceWriteIdentity{execUser: "1000", uid: 1000}, volumeWorkspaceWriteIdentityFromConfigUser("1000"))+ require.Equal(t, volumeWorkspaceWriteIdentity{execUser: "1000:1000", uid: 1000, gid: 1000}, volumeWorkspaceWriteIdentityFromConfigUser(" 1000:1000 "))+ require.Equal(t, volumeWorkspaceWriteIdentity{execUser: "1000:0", uid: 1000, gid: 0}, volumeWorkspaceWriteIdentityFromConfigUser("1000:0"))}func TestVolumeWorkspaceWritesUseRuntimeIdentity(t *testing.T) {- type execRequestInternal struct {+ type execRequest struct {user stringcmd []string}- var execRequests []execRequestInternal+ var execRequests []execRequestvar archiveQueries []stringvar archiveHeaders []*tar.Headerserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -361,22 +356,22 @@ func TestVolumeWorkspaceWritesUseRuntimeIdentity(t *testing.T) {}))t.Cleanup(server.Close)- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)applyResponse, err := json.Marshal(types.ApplyResponse{Version: types.ProtocolVersion, Applied: 1})require.NoError(t, err)service := NewService(Dependencies{Docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),Exec: func(_ context.Context, _, execUser string, cmd []string) (string, string, error) {- execRequests = append(execRequests, execRequestInternal{user: execUser, cmd: cmd})+ execRequests = append(execRequests, execRequest{user: execUser, cmd: cmd})return string(applyResponse), "", nil},})- identity := volumeWorkspaceWriteIdentityFromConfigUserInternal("1000:1000")+ identity := volumeWorkspaceWriteIdentityFromConfigUser("1000:1000")uploadIndex := 0changes := []volumetypes.WorkspaceFileChange{{Operation: volumetypes.FileOpCreateFile, RelativePath: "config.txt", UploadIndex: &uploadIndex}}- stagedFiles, err := service.stageVolumeWorkspaceChangesInternal(t.Context(), dockerClient, "helper", changes, map[int][]byte{0: []byte("content")}, identity)+ stagedFiles, err := service.stageVolumeWorkspaceChanges(t.Context(), dockerClient, "helper", changes, map[int][]byte{0: []byte("content")}, identity)require.NoError(t, err)- require.NoError(t, service.executeVolumeWorkspaceACFSBatchInternal(t.Context(), dockerClient, "helper", changes, stagedFiles, 0, identity))+ require.NoError(t, service.executeVolumeWorkspaceACFSBatch(t.Context(), dockerClient, "helper", changes, stagedFiles, 0, identity))require.Len(t, execRequests, 2)require.Empty(t, execRequests[0].user)@@ -417,7 +412,7 @@ func TestCreateVolumeWorkspaceMutationContainerUsesDedicatedBackupHelper(t *test}))t.Cleanup(server.Close)- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)acquireCalls := 0service := NewService(Dependencies{Docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),@@ -425,7 +420,7 @@ func TestCreateVolumeWorkspaceMutationContainerUsesDedicatedBackupHelper(t *testacquireCalls++return "cached-helper", func() {}, nil},- HelperImage: func(context.Context, *client.Client) (string, error) {+ HelperImage: func(context.Context) (string, error) {return volumehelper.ToolsImage(""), nil},BackupStorageMount: func(_ context.Context, _ *client.Client, target string, readOnly bool) (mount.Mount, error) {@@ -433,7 +428,7 @@ func TestCreateVolumeWorkspaceMutationContainerUsesDedicatedBackupHelper(t *test},})- containerID, cleanup, err := service.createVolumeWorkspaceMutationContainerInternal(t.Context(), "workspace-volume", true)+ containerID, cleanup, err := service.createVolumeWorkspaceMutationContainer(t.Context(), "workspace-volume", true)require.NoError(t, err)require.Equal(t, "restore-helper", containerID)require.Equal(t, 1, createCalls)@@ -508,7 +503,7 @@ acfs apply --root /volume --staging /tmp/staging --manifest manifest.json >/dev/sh -c "$1" sh dest/b.txt /tmp/backup.tar >/dev/nullacfs remove --root /volume --path /dest/b.txttar -xf /tmp/backup.tar -C /volume/dest-head -c 7 /volume/dest/b.txt`, volumeWorkspaceBackupCreateScriptInternal)+head -c 7 /volume/dest/b.txt`, volumeWorkspaceBackupCreateScript)require.Equal(t, "updated", restored)runInVolume(volumeName, `mkdir -p "/volume/Test Folder"@@ -517,16 +512,16 @@ printf spaced > "/volume/Test Folder/file.txt"`)sh -c "$1" sh "Test Folder" /tmp/space-backup.tar >/dev/nullrm -rf "/volume/Test Folder"tar -xf /tmp/space-backup.tar -C /volume-head -c 6 "/volume/Test Folder/file.txt"`, volumeWorkspaceBackupCreateScriptInternal)+head -c 6 "/volume/Test Folder/file.txt"`, volumeWorkspaceBackupCreateScript)require.Equal(t, "spaced", spacedRestored)missing := runInVolume(volumeName, `set -esh -c "$1" sh test.txt /tmp/missing-backup.tar-if stat -c '%A' -- /tmp/missing-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScriptInternal)+if stat -c '%A' -- /tmp/missing-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScript)require.Equal(t, "absent\x00test.txt\x00", missing)missingNested := runInVolume(volumeName, `set -esh -c "$1" sh "New Folder/test.txt" /tmp/missing-nested-backup.tar-if stat -c '%A' -- /tmp/missing-nested-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScriptInternal)+if stat -c '%A' -- /tmp/missing-nested-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScript)require.Equal(t, "absent\x00New Folder\x00", missingNested)}diff --git a/backend/internal/volume/helpers.go b/backend/internal/volume/helpers.goindex b6b77dea1..02303065d 100644--- a/backend/internal/volume/helpers.go+++ b/backend/internal/volume/helpers.go@@ -9,6 +9,7 @@ import ("fmt""io""log/slog"+ "slices""strings""time"@@ -27,10 +28,8 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper")-// volumeHelper tracks a reused helper container and the last-// time it serviced a request, so idle helpers can be reaped. inUse counts the-// requests currently holding the helper: the reaper must not remove a helper-// mid-download just because it was acquired longer ago than the idle timeout.+// volumeHelper tracks a reused helper container and its last use for idle reaping.+// inUse counts active holds so the reaper never removes a helper mid-request.type volumeHelper struct {id stringlastUsedAt time.Time@@ -38,7 +37,7 @@ type volumeHelper struct {protocol int}-func (s *VolumeService) requireVolumeHelperACFSInternal(ctx context.Context, volumeName, containerID string) error {+func (s *VolumeService) requireVolumeHelperACFS(ctx context.Context, volumeName, containerID string) error {s.helperMu.Lock()helper := s.helperByVolume[volumeName]if helper != nil && helper.id == containerID && helper.protocol >= types.ProtocolVersion {@@ -47,7 +46,7 @@ func (s *VolumeService) requireVolumeHelperACFSInternal(ctx context.Context, vol}s.helperMu.Unlock()- stdout, stderr, err := s.execInContainerInternal(ctx, containerID, "", []string{"acfs", "version"})+ stdout, stderr, err := s.execInContainer(ctx, containerID, "", []string{"acfs", "version"})if err != nil {return fmt.Errorf("%s: %w", "volume workspace requires an ACFS-capable tools image: "+strings.TrimSpace(stderr), err)}@@ -67,7 +66,7 @@ func (s *VolumeService) requireVolumeHelperACFSInternal(ctx context.Context, volreturn nil}-func (s *VolumeService) toolsImageInternal() string {+func (s *VolumeService) toolsImage() string {registry := ""if s.settingsService != nil {registry = s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value@@ -75,52 +74,31 @@ func (s *VolumeService) toolsImageInternal() string {return volumehelper.ToolsImage(registry)}-func isUnlabeledVolumeHelperContainerInternal(c container.Summary) bool {+func isUnlabeledVolumeHelperContainer(c container.Summary) bool {if internal, _ := kit.ParseBool(c.Labels[libarcane.InternalResourceLabel]); !internal {return false}command := strings.ToLower(c.Command)- if !strings.Contains(command, "sleep") || !strings.Contains(command, "infinity") {- return false- }-- for _, mounted := range c.Mounts {- if mounted.Destination == "/volume" {- return true- }- }-- return false+ return strings.Contains(command, "sleep") && strings.Contains(command, "infinity") && slices.ContainsFunc(c.Mounts, func(mounted container.MountPoint) bool {+ return mounted.Destination == "/volume"+ })}-func isVolumeHelperContainerInternal(c container.Summary) bool {- if isUnlabeledVolumeHelperContainerInternal(c) {- return true- }+func isVolumeHelperContainer(c container.Summary) bool {if internal, _ := kit.ParseBool(c.Labels[libarcane.InternalResourceLabel]); !internal {return false}-- return strings.EqualFold(c.Labels[volumehelper.ContainerLabel], "true")+ return strings.EqualFold(c.Labels[volumehelper.ContainerLabel], "true") || isUnlabeledVolumeHelperContainer(c)}-func (s *VolumeService) getVolumeHelperImageInternal(ctx context.Context, dockerClient *client.Client) (string, error) {+func (s *VolumeService) getVolumeHelperImage(ctx context.Context) (string, error) {slog.DebugContext(ctx, "volume service: resolve helper image")- var err error- if dockerClient == nil {- if s.dockerService == nil {- return "", errors.New("docker service unavailable")- }- dockerClient, err = s.dockerService.GetClient(ctx)- if err != nil {- return "", fmt.Errorf("failed to get docker client: %w", err)- }- }-- toolsImage := s.toolsImageInternal()+ toolsImage := s.toolsImage()- if _, imageInspectErr := dockerClient.ImageInspect(ctx, toolsImage); imageInspectErr == nil {+ if _, imageInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {+ return dockerClient.ImageInspect(ctx, toolsImage)+ }); imageInspectErr == nil {slog.InfoContext(ctx, "volume service: helper image strategy selected", "strategy", "tools-local", "image", toolsImage)return toolsImage, nil}@@ -139,6 +117,10 @@ func (s *VolumeService) getVolumeHelperImageInternal(ctx context.Context, dockerslog.WarnContext(ctx, "volume service: image service unavailable, attempting arcane fallback")}+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ return "", fmt.Errorf("failed to get docker client: %w", err)+ }if fallback, ok := volumehelper.ResolveArcaneRuntimeImage(ctx, dockerClient).Get(); ok {slog.InfoContext(ctx, "volume service: helper image strategy selected", "strategy", "arcane-fallback", "source", fallback.Source, "image", fallback.Image)return fallback.Image, nil@@ -147,37 +129,27 @@ func (s *VolumeService) getVolumeHelperImageInternal(ctx context.Context, dockerreturn "", fmt.Errorf("failed to resolve helper image: tools image unavailable and arcane fallback not found: %w", pullErr)}-func (s *VolumeService) acquireVolumeHelperInternal(ctx context.Context, volumeName string) (string, func(), error) {+func (s *VolumeService) acquireVolumeHelper(ctx context.Context, volumeName string) (string, func(), error) {slog.DebugContext(ctx, "volume service: acquire helper container", "volume", volumeName)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return "", nil, err- }- // Helpers are shared and reaped when idle; the caller's cleanup- // releases its in-use hold instead of removing the container. The resolve →- // acquire gap is racy against the reaper, so retry a resolve whose helper- // was reaped before this caller could take its hold.+ // The returned cleanup releases a hold on the shared helper; retry when the reaper wins the resolve-to-acquire race.for range 3 {- containerID, resolveHelperErr := s.resolveHelperInternal(ctx, dockerClient, volumeName)+ containerID, resolveHelperErr := s.resolveHelper(ctx, volumeName)if resolveHelperErr != nil {return "", nil, resolveHelperErr}- if release, ok := s.acquireHelperInternal(volumeName, containerID); ok {+ if release, ok := s.acquireHelper(volumeName, containerID); ok {return containerID, release, nil}}return "", nil, errors.New("failed to acquire volume helper container")}-// resolveHelperInternal returns the shared helper container ID-// for volumeName, creating it if needed. singleflight collapses concurrent-// misses: without it both requests create a helper and the loser is orphaned,-// holding the volume mount with no cleanup path until Arcane restarts. The-// creation itself can pull an image, so it must not run under helperMu.-func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (string, error) {+// resolveHelper returns volumeName's shared helper, creating it once via singleflight so concurrent misses never orphan one.+// Creation can pull an image, so it runs outside helperMu.+func (s *VolumeService) resolveHelper(ctx context.Context, volumeName string) (string, error) {resultCh := s.helperGroup.DoChan(volumeName, func() (any, error) {- helper, err := s.getReusableHelperInternal(ctx, dockerClient, volumeName)+ helper, err := s.getReusableHelper(ctx, volumeName)if err != nil {return nil, err}@@ -185,15 +157,11 @@ func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClientreturn containerID, nil}- // Detached from the caller's context so a client that walks away- // mid-create doesn't abort the helper the other waiters are blocked on,- // but still bounded: WithoutCancel alone would let a hung daemon or- // image pull wedge this singleflight key forever, queueing every later- // workspace request for the volume behind it with no recovery until restart.+ // Detached so one caller leaving does not abort the shared create, but bounded so a hung pull cannot wedge the key.createCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerImagePull)defer cancel()- containerID, _, createErr := s.startHelperContainerInternal(createCtx, dockerClient, volumeName)+ containerID, _, createErr := s.startHelperContainer(createCtx, volumeName)if createErr != nil {return nil, createErr}@@ -204,9 +172,7 @@ func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClientreturn containerID, nil})- // DoChan instead of Do so a caller whose request is canceled unblocks- // immediately; the shared create keeps running for the remaining waiters- // (and lands in helperByVolume for the next request even if none remain).+ // DoChan lets a canceled caller return at once while the shared create finishes for the others.var shared anyselect {case <-ctx.Done():@@ -225,13 +191,9 @@ func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClientreturn containerID, nil}-// acquireHelperInternal takes an in-use hold on volumeName's helper, reporting-// false when the helper has been removed (e.g. reaped) since it was resolved.-// The returned release drops the hold and refreshes the idle clock, so the idle-// timeout is measured from the end of the last request, not its start — a-// download running longer than the timeout used to have its helper reaped out-// from under it, truncating the stream.-func (s *VolumeService) acquireHelperInternal(volumeName, containerID string) (func(), bool) {+// acquireHelper takes a hold on volumeName's helper, or reports false if it was removed since being resolved.+// Release refreshes the idle clock so the timeout counts from the end of the last request.+func (s *VolumeService) acquireHelper(volumeName, containerID string) (func(), bool) {s.helperMu.Lock()defer s.helperMu.Unlock()@@ -253,11 +215,9 @@ func (s *VolumeService) acquireHelperInternal(volumeName, containerID string) (f}, true}-// startHelperContainerInternal creates and starts a volume helper container and-// returns a cleanup that removes it. Tracking of reusable helpers is-// the caller's responsibility.-func (s *VolumeService) startHelperContainerInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (string, func(), error) {- helperImage, err := s.getVolumeHelperImageInternal(ctx, dockerClient)+// startHelperContainer creates and starts a helper container and returns a cleanup that removes it.+func (s *VolumeService) startHelperContainer(ctx context.Context, volumeName string) (string, func(), error) {+ helperImage, err := s.getVolumeHelperImage(ctx)if err != nil {return "", nil, err}@@ -271,32 +231,38 @@ func (s *VolumeService) startHelperContainerInternal(ctx context.Context, dockerhostConfig := volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume"}, nil)- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: config,- HostConfig: hostConfig,+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: config,+ HostConfig: hostConfig,+ })})if err != nil {return "", nil, fmt.Errorf("failed to create temp container: %w", err)}- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {startErr := fmt.Errorf("failed to start temp container: %w", containerStartErr)- cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)- defer cancel()- if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {+ if _, cleanupErr := s.dockerService.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ }); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {return "", nil, errors.Join(startErr, fmt.Errorf("failed to remove temp container after start failure: %w", cleanupErr))}return "", nil, startErr}cleanup := func() {- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())+ _, _ = s.dockerService.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())+ })}return resp.ID, cleanup, nil}-func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (mo.Option[string], error) {+func (s *VolumeService) getReusableHelper(ctx context.Context, volumeName string) (mo.Option[string], error) {s.helperMu.Lock()helper := s.helperByVolume[volumeName]s.helperMu.Unlock()@@ -304,19 +270,22 @@ func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClireturn mo.None[string](), nil}- inspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, helper.id, client.ContainerInspectOptions{})+ inspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, helper.id, client.ContainerInspectOptions{})+ })if err != nil && !errdefs.IsNotFound(err) {return mo.None[string](), fmt.Errorf("failed to inspect volume helper container: %w", err)}- if err == nil && (inspect.Container.State == nil || !inspect.Container.State.Running) {- removeCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI)- _, removeErr := dockerClient.ContainerRemove(removeCtx, helper.id, volumehelper.RemoveOptions())- cancel()+ running := err == nil && kit.FromPtr(inspect.Container.State).Running+ if err == nil && !running {+ _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, helper.id, volumehelper.RemoveOptions())+ })if removeErr != nil && !errdefs.IsNotFound(removeErr) {return mo.None[string](), fmt.Errorf("failed to remove stopped volume helper container: %w", removeErr)}}- if err != nil || inspect.Container.State == nil || !inspect.Container.State.Running {+ if !running {s.helperMu.Lock()if s.helperByVolume[volumeName] == helper {delete(s.helperByVolume, volumeName)@@ -325,14 +294,13 @@ func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClireturn mo.None[string](), nil}- s.touchHelperInternal(volumeName)+ s.touchHelper(volumeName)return mo.Some(helper.id), nil}-// touchHelperInternal records that the helper for volumeName just serviced a-// request, resetting its idle clock. No-op if the entry is gone.-func (s *VolumeService) touchHelperInternal(volumeName string) {+// touchHelper resets the idle clock of volumeName's helper, if any.+func (s *VolumeService) touchHelper(volumeName string) {s.helperMu.Lock()defer s.helperMu.Unlock()if helper := s.helperByVolume[volumeName]; helper != nil {@@ -341,12 +309,6 @@ func (s *VolumeService) touchHelperInternal(volumeName string) {}func (s *VolumeService) CleanupHelperContainers(ctx context.Context) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- slog.WarnContext(ctx, "failed to get docker client for helper cleanup", "error", err)- return- }-s.helperMu.Lock()helperIDs := make([]string, 0, len(s.helperByVolume))for _, helper := range s.helperByVolume {@@ -358,32 +320,27 @@ func (s *VolumeService) CleanupHelperContainers(ctx context.Context) {s.helperMu.Unlock()for _, containerID := range helperIDs {- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions()); containerRemoveErr != nil {+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions())+ }); containerRemoveErr != nil {slog.WarnContext(ctx, "failed to remove helper container", "containerId", containerID, "error", containerRemoveErr.Error())}}}-// ReapIdleHelpers removes reused helper containers that have-// not serviced a request within idleTimeout. It is map-driven (orphaned helpers-// not tracked in helperByVolume are left to the startup orphan sweep). Entries-// are removed from the map before the container is removed, so a concurrent-// request simply gets a cache miss and re-creates a fresh helper.+// ReapIdleHelpers removes tracked helpers idle for idleTimeout; untracked orphans are left to the startup sweep.func (s *VolumeService) ReapIdleHelpers(ctx context.Context, idleTimeout time.Duration) (int, error) {if idleTimeout <= 0 {return 0, nil}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return 0, fmt.Errorf("failed to get docker client for idle helper reap: %w", err)- }-- staleIDs := s.collectStaleHelperIDsInternal(time.Now(), idleTimeout)+ staleIDs := s.collectStaleHelperIDs(time.Now(), idleTimeout)removed := 0for _, containerID := range staleIDs {- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions()); containerRemoveErr != nil {+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions())+ }); containerRemoveErr != nil {slog.WarnContext(ctx, "failed to remove idle helper container", "containerId", containerID, "error", containerRemoveErr.Error())continue}@@ -393,12 +350,9 @@ func (s *VolumeService) ReapIdleHelpers(ctx context.Context, idleTimeout time.Dureturn removed, nil}-// collectStaleHelperIDsInternal removes idle (and any nil) entries from the helper-// map and returns the container IDs that should be removed. Pure map/mutex-// bookkeeping with no Docker calls, so it can be unit-tested directly. Entries are-// dropped before their containers are removed so a concurrent request gets a clean-// cache miss and re-creates a fresh helper.-func (s *VolumeService) collectStaleHelperIDsInternal(now time.Time, idleTimeout time.Duration) []string {+// collectStaleHelperIDs drops idle and nil entries and returns their container IDs.+// Entries go before the containers so a concurrent request re-creates a fresh helper.+func (s *VolumeService) collectStaleHelperIDs(now time.Time, idleTimeout time.Duration) []string {staleIDs := make([]string, 0)s.helperMu.Lock()defer s.helperMu.Unlock()@@ -407,9 +361,7 @@ func (s *VolumeService) collectStaleHelperIDsInternal(now time.Time, idleTimeoutdelete(s.helperByVolume, volumeName)continue}- // A helper with active holds is serving a request right now (e.g. a- // download streaming for longer than the idle timeout); it becomes- // reapable once released, since release refreshes lastUsedAt.+ // Held helpers are serving a request; release refreshes lastUsedAt.if helper.inUse > 0 {continue}@@ -421,34 +373,28 @@ func (s *VolumeService) collectStaleHelperIDsInternal(now time.Time, idleTimeoutreturn staleIDs}-// StopHelper removes the reused helper for a single volume, if-// one exists. It is idempotent: stopping a volume with no active helper returns-// nil.+// StopHelper removes the reused helper for a single volume; it is a no-op when none exists.func (s *VolumeService) StopHelper(ctx context.Context, volumeName string) error {if strings.TrimSpace(volumeName) == "" {return nil}- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return fmt.Errorf("failed to get docker client for helper stop: %w", err)- }-- containerID := s.takeHelperIDInternal(volumeName)+ containerID := s.takeHelperID(volumeName)if containerID == "" {return nil}- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions()); containerRemoveErr != nil {+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions())+ }); containerRemoveErr != nil {return fmt.Errorf("failed to remove helper container: %w", containerRemoveErr)}return nil}-// takeHelperIDInternal removes the helper entry for volumeName and returns its-// container ID, or "" if there was none. Pure map/mutex bookkeeping.-func (s *VolumeService) takeHelperIDInternal(volumeName string) string {+// takeHelperID removes volumeName's helper entry and returns its container ID, or "".+func (s *VolumeService) takeHelperID(volumeName string) string {s.helperMu.Lock()defer s.helperMu.Unlock()helper := s.helperByVolume[volumeName]@@ -462,23 +408,22 @@ func (s *VolumeService) takeHelperIDInternal(volumeName string) string {func (s *VolumeService) CleanupOrphanedVolumeHelpers(ctx context.Context) (int, error) {slog.DebugContext(ctx, "volume service: cleanup orphaned volume helper containers")- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return 0, fmt.Errorf("failed to get docker client for orphan helper cleanup: %w", err)- }-- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ containers, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})+ })if err != nil {return 0, fmt.Errorf("failed to list containers for orphan helper cleanup: %w", err)}removedCount := 0for _, c := range containers.Items {- if !isVolumeHelperContainerInternal(c) {+ if !isVolumeHelperContainer(c) {continue}- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, c.ID, volumehelper.RemoveOptions()); containerRemoveErr != nil {+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {+ return dockerClient.ContainerRemove(ctx, c.ID, volumehelper.RemoveOptions())+ }); containerRemoveErr != nil {slog.WarnContext(ctx,"volume service: failed to remove orphaned volume helper container",@@ -495,16 +440,7 @@ func (s *VolumeService) CleanupOrphanedVolumeHelpers(ctx context.Context) (int,return removedCount, nil}-func (s *VolumeService) removeHelperEntry(volumeName string) {- if strings.TrimSpace(volumeName) == "" {- return- }- s.helperMu.Lock()- delete(s.helperByVolume, volumeName)- s.helperMu.Unlock()-}--func (s *VolumeService) execInContainerInternal(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error) {+func (s *VolumeService) execInContainer(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error) {slog.DebugContext(ctx, "volume service: exec in container", "containerId", containerID, "user", execUser, "cmd", cmd)dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {diff --git a/backend/internal/volume/helpers_test.go b/backend/internal/volume/helpers_test.goindex 10268b803..28e48e08d 100644--- a/backend/internal/volume/helpers_test.go+++ b/backend/internal/volume/helpers_test.go@@ -17,7 +17,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper")-func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {+func newVolumeServiceTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {t.Helper()dockerClient, err := client.New(client.WithHost(server.URL),@@ -29,7 +29,7 @@ func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Serreturn dockerClient}-func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {+func TestCreateTempContainerReusesWritableHelper(t *testing.T) {var createCalls, startCalls, removeCalls intvar createRequest struct {NetworkDisabled bool `json:"NetworkDisabled"`@@ -73,13 +73,13 @@ func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {t.Cleanup(server.Close)service := &VolumeService{- dockerService: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeServiceTestDockerClientInternal(t, server)),+ dockerService: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeServiceTestDockerClient(t, server)),helperByVolume: make(map[string]*volumeHelper),}- firstID, releaseFirst, err := service.acquireVolumeHelperInternal(t.Context(), "workspace-volume")+ firstID, releaseFirst, err := service.acquireVolumeHelper(t.Context(), "workspace-volume")require.NoError(t, err)- secondID, releaseSecond, err := service.acquireVolumeHelperInternal(t.Context(), "workspace-volume")+ secondID, releaseSecond, err := service.acquireVolumeHelper(t.Context(), "workspace-volume")require.NoError(t, err)require.Equal(t, "helper-1", firstID)@@ -93,7 +93,7 @@ func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {require.Equal(t, 2, service.helperByVolume["workspace-volume"].inUse)service.helperByVolume["workspace-volume"].lastUsedAt = time.Now().Add(-time.Hour)- require.Empty(t, service.collectStaleHelperIDsInternal(time.Now(), time.Minute))+ require.Empty(t, service.collectStaleHelperIDs(time.Now(), time.Minute))require.Contains(t, service.helperByVolume, "workspace-volume")releaseFirst()@@ -101,7 +101,7 @@ func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {require.Zero(t, service.helperByVolume["workspace-volume"].inUse)}-func TestIsUnlabeledVolumeHelperContainerInternal(t *testing.T) {+func TestIsUnlabeledVolumeHelperContainer(t *testing.T) {tests := []struct {name stringsummary container.Summary@@ -137,12 +137,12 @@ func TestIsUnlabeledVolumeHelperContainerInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- require.Equal(t, tt.want, isUnlabeledVolumeHelperContainerInternal(tt.summary))+ require.Equal(t, tt.want, isUnlabeledVolumeHelperContainer(tt.summary))})}}-func TestIsVolumeHelperContainerInternal_UsesExplicitHelperLabel(t *testing.T) {+func TestIsVolumeHelperContainer_UsesExplicitHelperLabel(t *testing.T) {tests := []struct {name stringsummary container.Summary@@ -187,12 +187,12 @@ func TestIsVolumeHelperContainerInternal_UsesExplicitHelperLabel(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- require.Equal(t, tt.want, isVolumeHelperContainerInternal(tt.summary))+ require.Equal(t, tt.want, isVolumeHelperContainer(tt.summary))})}}-func TestCollectStaleHelperIDsInternal(t *testing.T) {+func TestCollectStaleHelperIDs(t *testing.T) {now := time.Now()s := &VolumeService{helperByVolume: map[string]*volumeHelper{@@ -203,7 +203,7 @@ func TestCollectStaleHelperIDsInternal(t *testing.T) {},}- stale := s.collectStaleHelperIDsInternal(now, 10*time.Minute)+ stale := s.collectStaleHelperIDs(now, 10*time.Minute)require.ElementsMatch(t, []string{"c-stale", "c-atedge"}, stale,"helpers idle >= timeout (and exactly at the edge) should be collected")@@ -216,7 +216,7 @@ func TestCollectStaleHelperIDsInternal(t *testing.T) {// A helper serving a request longer than the idle timeout (e.g. a slow download)// must not be reaped mid-stream; it becomes collectible once released, with the// idle clock measured from the release.-func TestCollectStaleHelperIDsInternalSkipsInUseHelpers(t *testing.T) {+func TestCollectStaleHelperIDsSkipsInUseHelpers(t *testing.T) {now := time.Now()s := &VolumeService{helperByVolume: map[string]*volumeHelper{@@ -224,26 +224,26 @@ func TestCollectStaleHelperIDsInternalSkipsInUseHelpers(t *testing.T) {},}- release, ok := s.acquireHelperInternal("vol-a", "c-a")+ release, ok := s.acquireHelper("vol-a", "c-a")require.True(t, ok)- require.Empty(t, s.collectStaleHelperIDsInternal(now, 10*time.Minute),+ require.Empty(t, s.collectStaleHelperIDs(now, 10*time.Minute),"an in-use helper must survive the reaper regardless of lastUsedAt")require.Contains(t, s.helperByVolume, "vol-a")release()- require.Empty(t, s.collectStaleHelperIDsInternal(now, 10*time.Minute),+ require.Empty(t, s.collectStaleHelperIDs(now, 10*time.Minute),"release refreshes the idle clock, so the helper is fresh again")- stale := s.collectStaleHelperIDsInternal(now.Add(11*time.Minute), 10*time.Minute)+ stale := s.collectStaleHelperIDs(now.Add(11*time.Minute), 10*time.Minute)require.Equal(t, []string{"c-a"}, stale)// Acquiring a helper that was reaped (or replaced) since resolve must fail// so the caller re-resolves instead of using a dead container.- _, ok = s.acquireHelperInternal("vol-a", "c-a")+ _, ok = s.acquireHelper("vol-a", "c-a")require.False(t, ok)}-func TestTakeHelperIDInternal(t *testing.T) {+func TestTakeHelperID(t *testing.T) {s := &VolumeService{helperByVolume: map[string]*volumeHelper{"vol-a": {id: "c-a", lastUsedAt: time.Now()},@@ -251,15 +251,15 @@ func TestTakeHelperIDInternal(t *testing.T) {}// Present: returns id and removes the entry.- require.Equal(t, "c-a", s.takeHelperIDInternal("vol-a"))+ require.Equal(t, "c-a", s.takeHelperID("vol-a"))require.NotContains(t, s.helperByVolume, "vol-a")// Absent (idempotent): returns "" without panicking.- require.Empty(t, s.takeHelperIDInternal("vol-a"))- require.Empty(t, s.takeHelperIDInternal("never-existed"))+ require.Empty(t, s.takeHelperID("vol-a"))+ require.Empty(t, s.takeHelperID("never-existed"))}-func TestTouchHelperInternal(t *testing.T) {+func TestTouchHelper(t *testing.T) {old := time.Now().Add(-30 * time.Minute)s := &VolumeService{helperByVolume: map[string]*volumeHelper{@@ -267,11 +267,11 @@ func TestTouchHelperInternal(t *testing.T) {},}- s.touchHelperInternal("vol-a")+ s.touchHelper("vol-a")require.True(t, s.helperByVolume["vol-a"].lastUsedAt.After(old),"touch should reset the idle clock forward")// Missing volume is a no-op (must not panic or create an entry).- s.touchHelperInternal("missing")+ s.touchHelper("missing")require.NotContains(t, s.helperByVolume, "missing")}diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.goindex 8b9b8cd3e..1bd364666 100644--- a/backend/internal/volume/service.go+++ b/backend/internal/volume/service.go@@ -8,6 +8,7 @@ import ("log/slog""net/url""regexp"+ "slices""strings""sync""time"@@ -41,7 +42,6 @@ import (volumebackup "github.com/getarcaneapp/arcane/backend/v2/internal/volume/children/backup""github.com/getarcaneapp/arcane/backend/v2/internal/volume/children/workspace""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumes""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"@@ -112,7 +112,7 @@ func NewVolumeService(helperByVolume: make(map[string]*volumeHelper),}backupDeps := volumebackup.Dependencies{- Store: service.backupStoreInternal(),+ Store: service.backupStore(),DB: db,Docker: dockerService,Events: eventService,@@ -122,9 +122,9 @@ func NewVolumeService(RecoveryKeys: recoveryKeys,Locks: &service.workspaceLocks,AlreadyRunning: ErrVolumeBackupAlreadyRunning,- HelperImage: service.getVolumeHelperImageInternal,- AcquireHelper: service.acquireVolumeHelperInternal,- Exec: service.execInContainerInternal,+ HelperImage: service.getVolumeHelperImage,+ AcquireHelper: service.acquireVolumeHelper,+ Exec: service.execInContainer,BackupVolumeName: backupVolumeName,EncryptionKey: encryptionKey,}@@ -136,10 +136,10 @@ func NewVolumeService(Docker: dockerService,Events: eventService,Locks: &service.workspaceLocks,- AcquireHelper: service.acquireVolumeHelperInternal,- RequireACFS: service.requireVolumeHelperACFSInternal,- Exec: service.execInContainerInternal,- HelperImage: service.getVolumeHelperImageInternal,+ AcquireHelper: service.acquireVolumeHelper,+ RequireACFS: service.requireVolumeHelperACFS,+ Exec: service.execInContainer,+ HelperImage: service.getVolumeHelperImage,BackupStorageMount: service.backup.StorageMount,ArchiveFileTarget: service.backup.ArchiveFileTarget,RestoreArchiveMembers: service.backup.RestoreArchiveMembers,@@ -171,19 +171,14 @@ func (s *VolumeService) HasEnabledBackupPolicy(ctx context.Context, volumeName s}// CreateSystemManagedBackup runs the existing volume backup workflow with a transient centralized policy.-func (- s *VolumeService,-) CreateSystemManagedBackup(+func (s *VolumeService) CreateSystemManagedBackup(ctx context.Context,volumeName string,user usertypes.Actor,trigger VolumeBackupTrigger,policyID string,policy backuptypes.UpdateBackupPolicy,-) (- *VolumeBackup,- error,-) {+) (*VolumeBackup, error) {entry, err := s.backup.CreateSystemManagedBackup(ctx, volumeName, user, string(trigger), policyID, policy)if entry == nil {return nil, err@@ -224,7 +219,7 @@ func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context) error {return s.backup.MigrateRepositoryPasswords(ctx)}-func (s *VolumeService) backupStoreInternal() volumebackup.Store {+func (s *VolumeService) backupStore() volumebackup.Store {return volumebackup.Store{Create: func(ctx context.Context, record *volumetypes.Backup) error {entry := volumeBackupFromRecordInternal(record)@@ -277,7 +272,7 @@ func (s *VolumeService) backupStoreInternal() volumebackup.Store {}return entry.recordInternal(), nil},- List: s.listBackupsInternal,+ List: s.listBackups,RemoteSnapshotIDs: func(ctx context.Context, destinationID string) ([]string, error) {var knownIDs []stringerr := s.db.WithContext(ctx).Model(&VolumeBackup{}).@@ -288,8 +283,8 @@ func (s *VolumeService) backupStoreInternal() volumebackup.Store {}}-// listBackupsInternal pages one volume's backup history; both or neither management type selected means no type filter.-func (s *VolumeService) listBackupsInternal(ctx context.Context, volumeName string, params pagination.QueryParams) ([]volumetypes.Backup, int64, error) {+// listBackups pages one volume's backup history; both or neither management type selected means no type filter.+func (s *VolumeService) listBackups(ctx context.Context, volumeName string, params pagination.QueryParams) ([]volumetypes.Backup, int64, error) {query := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("volume_name = ?", volumeName)var system, managed boolfor value := range strings.SplitSeq(params.Filters["type"], ",") {@@ -373,33 +368,20 @@ func (s *VolumeService) listBackupsInternal(ctx context.Context, volumeName strifunc (s *VolumeService) GetVolumeByName(ctx context.Context, name string) (*volumetypes.Volume, error) {slog.DebugContext(ctx, "volume service: get volume", "volume", name)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- volResult, err := dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})+ volResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})+ })if err != nil {return nil, fmt.Errorf("volume not found: %w", err)}- vol := volResult.Volume-- localSettings := s.settingsService.GetSettingsConfig()- usageCtx, usageCancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer usageCancel()- if usageVolumes, ok := docker.GetVolumeUsageDataStaleWhileRevalidate(usageCtx, dockerClient).Get(); ok {- for _, uv := range usageVolumes {- if uv.Name == vol.Name && uv.UsageData != nil {- vol.UsageData = uv.UsageData- slog.DebugContext(ctx, "attached volume usage data", "volume", vol.Name, "sizeBytes", uv.UsageData.Size, "refCount", uv.UsageData.RefCount)- break- }- }- }-- v := volumetypes.NewSummary(vol)+ usage, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (mo.Option[[]volume.Volume], error) {+ return docker.GetVolumeUsageDataStaleWhileRevalidate(ctx, dockerClient), nil+ })+ v := volumetypes.NewSummary(enrichVolumesWithUsageData([]volume.Volume{volResult.Volume}, usage.OrEmpty())[0])- containerIDs, err := docker.GetContainersUsingVolume(ctx, dockerClient, name)+ containerIDs, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]string, error) {+ return docker.GetContainersUsingVolume(ctx, dockerClient, name)+ })if err != nil {slog.WarnContext(ctx, "failed to get containers using volume", "volume", name, "error", err.Error())} else {@@ -421,14 +403,18 @@ func (s *VolumeService) CreateVolume(ctx context.Context, options client.VolumeC}defer s.eventService.BeginDockerResourceSuppressionWindow("volume", options.Name, options.Name)()- created, err := dockerClient.VolumeCreate(ctx, options)+ created, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {+ return dockerClient.VolumeCreate(ctx, options)+ })if err != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", "", options.Name, user.ID, user.Username, "0", err, database.JSON{"action": "create", "driver": options.Driver})return nil, fmt.Errorf("failed to create volume: %w", err)}defer s.eventService.BeginDockerResourceSuppressionWindow("volume", created.Volume.Name, created.Volume.Name)()- vol, err := dockerClient.VolumeInspect(ctx, created.Volume.Name, client.VolumeInspectOptions{})+ vol, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, created.Volume.Name, client.VolumeInspectOptions{})+ })if err != nil {s.eventService.LogErrorEvent(ctx,@@ -482,8 +468,8 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boo}defer s.eventService.BeginDockerResourceSuppressionWindow("volume", name, name)()- if _, volumeRemoveErr := dockerClient.VolumeRemove(ctx, name, client.VolumeRemoveOptions{- Force: force,+ if _, volumeRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeRemoveResult, error) {+ return dockerClient.VolumeRemove(ctx, name, client.VolumeRemoveOptions{Force: force})}); volumeRemoveErr != nil {s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", name, name, user.ID, user.Username, "0", volumeRemoveErr, database.JSON{"action": "delete", "force": force})return fmt.Errorf("failed to remove volume: %w", volumeRemoveErr)@@ -497,7 +483,7 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force booslog.WarnContext(ctx, "could not log volume deletion action", "volume", name, "error", logErr.Error())}- s.removeHelperEntry(name)+ s.takeHelperID(name)s.backup.RemovePolicies(ctx, name)docker.InvalidateVolumeUsageCache(dockerClient)return nil@@ -524,18 +510,27 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (// - Without 'all' flag: Only removes anonymous (unnamed) volumes that are not in use// - With 'all=true' flag: Removes ALL unused volumes (both named and anonymous)// Note: Volumes are considered "in use" if referenced by any container (running or stopped)- volumePruneResult, err := dockerClient.VolumePrune(ctx, buildVolumePruneOptionsInternal(all))+ volumePruneResult, err := dockerClient.VolumePrune(ctx, client.VolumePruneOptions{+ All: all,+ Filters: make(client.Filters).Add("label!", internalVolumePruneFilterValue),+ })if err != nil {return nil, fmt.Errorf("failed to prune volumes: %w", err)}- metadata := buildVolumePruneMetadataInternal(all, len(volumePruneResult.Report.VolumesDeleted), volumePruneResult.Report.SpaceReclaimed)+ metadata := database.JSON{+ "action": "prune",+ "all": all,+ "volumesDeleted": len(volumePruneResult.Report.VolumesDeleted),+ "spaceReclaimed": volumePruneResult.Report.SpaceReclaimed,+ "internalVolumeFilterLabel": internalVolumePruneFilterValue,+ }if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeDelete, "", "bulk_prune", usertypes.SystemUser.ID, usertypes.SystemUser.Username, "0", metadata); logErr != nil {slog.WarnContext(ctx, "could not log volume prune action", "error", logErr.Error())}for _, volumeName := range volumePruneResult.Report.VolumesDeleted {- s.removeHelperEntry(volumeName)+ s.takeHelperID(volumeName)s.backup.RemovePolicies(ctx, volumeName)}@@ -547,35 +542,18 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (}, nil}-func buildVolumePruneOptionsInternal(all bool) client.VolumePruneOptions {- filters := make(client.Filters)- filters = filters.Add("label!", internalVolumePruneFilterValue)- return client.VolumePruneOptions{All: all, Filters: filters}-}--func buildVolumePruneMetadataInternal(all bool, volumesDeleted int, spaceReclaimed uint64) database.JSON {- return database.JSON{- "action": "prune",- "all": all,- "volumesDeleted": volumesDeleted,- "spaceReclaimed": spaceReclaimed,- "internalVolumeFilterLabel": internalVolumePruneFilterValue,- }-}-func (s *VolumeService) GetVolumeUsage(ctx context.Context, name string) (bool, []string, error) {slog.DebugContext(ctx, "volume service: get volume usage", "volume", name)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return false, nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- vol, err := dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})+ vol, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})+ })if err != nil {return false, nil, fmt.Errorf("volume not found: %w", err)}- containerIDs, err := docker.GetContainersUsingVolume(ctx, dockerClient, vol.Volume.Name)+ containerIDs, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]string, error) {+ return docker.GetContainersUsingVolume(ctx, dockerClient, vol.Volume.Name)+ })if err != nil {return false, nil, fmt.Errorf("failed to get containers using volume: %w", err)}@@ -594,16 +572,7 @@ type VolumeSizeData struct {// This is a slow operation as it calls Docker's DiskUsage API.func (s *VolumeService) GetVolumeSizes(ctx context.Context) (map[string]VolumeSizeData, error) {slog.DebugContext(ctx, "volume service: get volume sizes")- localSettings := s.settingsService.GetSettingsConfig()- apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer cancel()-- dockerClient, err := s.dockerService.GetClient(apiCtx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }-- usageVolumes, err := docker.GetVolumeUsageData(apiCtx, dockerClient)+ usageVolumes, err := s.dockerService.Call(ctx, docker.GetVolumeUsageData)if err != nil {return nil, fmt.Errorf("failed to get volume usage data: %w", err)}@@ -621,7 +590,7 @@ func (s *VolumeService) GetVolumeSizes(ctx context.Context) (map[string]VolumeSireturn result, nil}-func enrichVolumesWithUsageDataInternal(dockerVolumes, usageVolumes []volume.Volume) []volume.Volume {+func enrichVolumesWithUsageData(dockerVolumes, usageVolumes []volume.Volume) []volume.Volume {usageByName := make(map[string]*volume.UsageData, len(usageVolumes))for _, uv := range usageVolumes {if uv.Name == "" || uv.UsageData == nil {@@ -644,25 +613,24 @@ func enrichVolumesWithUsageDataInternal(dockerVolumes, usageVolumes []volume.Volreturn result}-func (s *VolumeService) buildVolumeContainerMapInternal(ctx context.Context) (map[string][]string, error) {- containers, err := s.dockerService.ListContainers(ctx)- if err != nil {- return nil, fmt.Errorf("failed to list containers: %w", err)- }-- volumeContainerMap := make(map[string][]string)- for _, c := range containers {- for _, m := range c.Mounts {- if m.Type == mount.TypeVolume && m.Name != "" {- volumeContainerMap[m.Name] = append(volumeContainerMap[m.Name], c.ID)+func (s *VolumeService) buildVolumePaginationConfig() pagination.Config[volumetypes.Volume] {+ createdSortFn := func(a, b volumetypes.Volume) int {+ parseCreatedAt := func(createdAt string) (time.Time, bool) {+ for _, layout := range []string{time.RFC3339Nano, time.RFC3339} {+ if parsed, err := time.Parse(layout, createdAt); err == nil {+ return parsed, true+ }}+ return time.Time{}, false+ }+ aTime, aOk := parseCreatedAt(a.CreatedAt)+ bTime, bOk := parseCreatedAt(b.CreatedAt)+ if aOk && bOk {+ return aTime.Compare(bTime)}+ return strings.Compare(a.CreatedAt, b.CreatedAt)}- return volumeContainerMap, nil-}--func (s *VolumeService) buildVolumePaginationConfig() pagination.Config[volumetypes.Volume] {return pagination.Config[volumetypes.Volume]{SearchAccessors: []pagination.SearchAccessor[volumetypes.Volume]{func(v volumetypes.Volume) (string, error) { return v.Name, nil },@@ -670,127 +638,62 @@ func (s *VolumeService) buildVolumePaginationConfig() pagination.Config[volumetyfunc(v volumetypes.Volume) (string, error) { return v.Mountpoint, nil },func(v volumetypes.Volume) (string, error) { return v.Scope, nil },},- SortBindings: s.buildVolumeSortBindingsInternal(),- FilterAccessors: buildVolumeFilterAccessorsInternal(),- }-}--func (s *VolumeService) buildVolumeSortBindingsInternal() []pagination.SortBinding[volumetypes.Volume] {- createdSortFn := s.compareVolumeCreatedInternal-- return []pagination.SortBinding[volumetypes.Volume]{- {- Key: "name",- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Name, b.Name) },- },- {- Key: "driver",- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Driver, b.Driver) },- },- {- Key: "mountpoint",- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Mountpoint, b.Mountpoint) },- },- {- Key: "scope",- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Scope, b.Scope) },- },- {- Key: "created",- Fn: createdSortFn,- },- {- Key: "createdAt",- Fn: createdSortFn,- },- {- Key: "inUse",- Fn: func(a, b volumetypes.Volume) int {- if a.InUse == b.InUse {- return 0- }- return kit.Ternary(a.InUse, -1, 1)+ SortBindings: []pagination.SortBinding[volumetypes.Volume]{+ {+ Key: "name",+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Name, b.Name) },+ },+ {+ Key: "driver",+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Driver, b.Driver) },+ },+ {+ Key: "mountpoint",+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Mountpoint, b.Mountpoint) },+ },+ {+ Key: "scope",+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Scope, b.Scope) },+ },+ {+ Key: "created",+ Fn: createdSortFn,+ },+ {+ Key: "createdAt",+ Fn: createdSortFn,+ },+ {+ Key: "inUse",+ Fn: func(a, b volumetypes.Volume) int {+ if a.InUse == b.InUse {+ return 0+ }+ return kit.Ternary(a.InUse, -1, 1)+ },+ },+ {+ Key: "size",+ Fn: func(a, b volumetypes.Volume) int {+ aSize := cmp.Or(a.Size, kit.FromPtr(a.UsageData).Size)+ bSize := cmp.Or(b.Size, kit.FromPtr(b.UsageData).Size)+ return cmp.Or(cmp.Compare(aSize, bSize), strings.Compare(a.Name, b.Name))+ },},},- {- Key: "size",- Fn: compareVolumeSizesInternal,- },- }-}--func compareVolumeSizesInternal(a, b volumetypes.Volume) int {- aSize := a.Size- bSize := b.Size-- if aSize == 0 && a.UsageData != nil {- aSize = a.UsageData.Size- }- if bSize == 0 && b.UsageData != nil {- bSize = b.UsageData.Size- }-- if aSize == bSize {- return strings.Compare(a.Name, b.Name)- }- return kit.Ternary(aSize < bSize, -1, 1)-}--func (s *VolumeService) compareVolumeCreatedInternal(a, b volumetypes.Volume) int {- aTime, aOk := parseVolumeCreatedAtInternal(a.CreatedAt).Get()- bTime, bOk := parseVolumeCreatedAtInternal(b.CreatedAt).Get()- if aOk && bOk {- if aTime.Before(bTime) {- return -1- }- return kit.Ternary(aTime.After(bTime), 1, 0)- }- return strings.Compare(a.CreatedAt, b.CreatedAt)-}--func parseVolumeCreatedAtInternal(createdAt string) mo.Option[time.Time] {- if createdAt == "" {- return mo.None[time.Time]()- }- if parsed, err := time.Parse(time.RFC3339Nano, createdAt); err == nil {- return mo.Some(parsed)- }- if parsed, err := time.Parse(time.RFC3339, createdAt); err == nil {- return mo.Some(parsed)- }- return mo.None[time.Time]()-}--func buildVolumeFilterAccessorsInternal() []pagination.FilterAccessor[volumetypes.Volume] {- return []pagination.FilterAccessor[volumetypes.Volume]{- {- Key: "inUse",- Fn: func(v volumetypes.Volume, filterValue string) bool {- return kit.Ternary(filterValue == "true", v.InUse, filterValue != "false" || !v.InUse)+ FilterAccessors: []pagination.FilterAccessor[volumetypes.Volume]{+ {+ Key: "inUse",+ Fn: func(v volumetypes.Volume, filterValue string) bool {+ return kit.Ternary(filterValue == "true", v.InUse, filterValue != "false" || !v.InUse)+ },},},}}-func calculateVolumeUsageCountsInternal(items []volumetypes.Volume) volumetypes.UsageCounts {- counts := volumetypes.UsageCounts{- Total: len(items),- }- for _, v := range items {- if v.InUse {- counts.Inuse++- } else {- counts.Unused++- }- }- return counts-}--// CountUsageFromSnapshot derives volume usage counts from an-// already-fetched volume and container listing. It mirrors the semantics of-// ListVolumesPaginated — internal volumes are excluded and a volume counts as-// in use when any container mounts it — so dashboard tile numbers agree with-// the volumes page.+// CountUsageFromSnapshot derives usage counts from fetched volumes and containers with ListVolumesPaginated's+// semantics (internal volumes excluded, mounted means in use) so dashboard tiles match the volumes page.func (s *VolumeService) CountUsageFromSnapshot(dockerVolumes []volume.Volume, containers []container.Summary) volumetypes.UsageCounts {inUse := make(map[string]struct{})for _, c := range containers {@@ -803,7 +706,7 @@ func (s *VolumeService) CountUsageFromSnapshot(dockerVolumes []volume.Volume, cocounts := volumetypes.UsageCounts{}for _, v := range dockerVolumes {- if s.isInternalVolumeInternal(volumetypes.NewSummary(v)) {+ if s.isInternalVolume(volumetypes.NewSummary(v)) {continue}counts.Total++@@ -821,7 +724,7 @@ func (s *VolumeService) IsBackupVolume(name string) bool {return strings.EqualFold(strings.TrimSpace(name), strings.TrimSpace(s.backupVolumeName))}-func (s *VolumeService) isInternalVolumeInternal(v volumetypes.Volume) bool {+func (s *VolumeService) isInternalVolume(v volumetypes.Volume) bool {if s.IsBackupVolume(v.Name) {return true}@@ -830,55 +733,33 @@ func (s *VolumeService) isInternalVolumeInternal(v volumetypes.Volume) bool {return internal}-var generatedAnonymousVolumeNameInternal = regexp.MustCompile(`^[a-f0-9]{64}$`)--func isAnonymousVolumeInternal(v volume.Volume) bool {- if _, ok := v.Labels["com.docker.volume.anonymous"]; ok {- return true- }- return generatedAnonymousVolumeNameInternal.MatchString(v.Name)-}--func mountedVolumeNamesInternal(mounts []container.MountPoint) map[string]struct{} {- names := make(map[string]struct{})- for _, item := range mounts {- if item.Type != mount.TypeVolume {- continue- }- name := cmp.Or(strings.TrimSpace(item.Name), strings.TrimSpace(item.Source))- if name != "" {- names[name] = struct{}{}- }- }- return names-}+var generatedAnonymousVolumeName = regexp.MustCompile(`^[a-f0-9]{64}$`)// ListBackupVolumeOptions returns the current non-internal volumes used by centralized backup selection.func (s *VolumeService) ListBackupVolumeOptions(ctx context.Context) ([]backuptypes.SystemVolumeBackupOption, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }- result, err := dockerClient.VolumeList(ctx, client.VolumeListOptions{})+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeListResult, error) {+ return dockerClient.VolumeList(ctx, client.VolumeListOptions{})+ })if err != nil {return nil, fmt.Errorf("failed to list Docker volumes: %w", err)}- arcaneVolumes := make(map[string]struct{})- if inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr == nil && inspect != nil {- arcaneVolumes = mountedVolumeNamesInternal(inspect.Mounts)- } else if inspectErr != nil {+ inspect, inspectErr := s.dockerService.Call(ctx, libarcane.InspectCurrentArcaneContainer)+ if inspectErr != nil {slog.DebugContext(ctx, "volume service: could not identify Arcane-mounted volumes for centralized backups", "error", inspectErr)}+ arcaneMounts := kit.FromPtr(inspect).Mountsoptions := make([]backuptypes.SystemVolumeBackupOption, 0, len(result.Items))for _, item := range result.Items {- if s.isInternalVolumeInternal(volumetypes.NewSummary(item)) {- continue- }- if _, isArcaneVolume := arcaneVolumes[item.Name]; isArcaneVolume {+ if s.isInternalVolume(volumetypes.NewSummary(item)) || slices.ContainsFunc(arcaneMounts, func(m container.MountPoint) bool {+ return m.Type == mount.TypeVolume && cmp.Or(strings.TrimSpace(m.Name), strings.TrimSpace(m.Source)) == item.Name+ }) {continue}+ _, anonymousLabel := item.Labels["com.docker.volume.anonymous"]options = append(options, backuptypes.SystemVolumeBackupOption{- Name: item.Name, Anonymous: isAnonymousVolumeInternal(item), Available: true,+ Name: item.Name,+ Anonymous: anonymousLabel || generatedAnonymousVolumeName.MatchString(item.Name),+ Available: true,})}return options, nil@@ -902,10 +783,6 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat"includeInternal",includeInternal,)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, pagination.Response{}, volumetypes.UsageCounts{}, fmt.Errorf("failed to connect to Docker: %w", err)- }// Run volume list and container list in parallel for better performancetype volumeListResult struct {@@ -920,19 +797,29 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginatvolChan := make(chan volumeListResult, 1)containerChan := make(chan containerMapResult, 1)- localSettings := s.settingsService.GetSettingsConfig()- apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer cancel()-- go func(ctx context.Context) {- volListBody, volumeListErr := dockerClient.VolumeList(ctx, client.VolumeListOptions{})+ go func() {+ volListBody, volumeListErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeListResult, error) {+ return dockerClient.VolumeList(ctx, client.VolumeListOptions{})+ })volChan <- volumeListResult{volumes: volListBody.Items, err: volumeListErr}- }(apiCtx)+ }()- go func(ctx context.Context) {- containerMap, buildVolumeContainerMapErr := s.buildVolumeContainerMapInternal(ctx)- containerChan <- containerMapResult{containerMap: containerMap, err: buildVolumeContainerMapErr}- }(apiCtx)+ go func() {+ containers, listErr := s.dockerService.ListContainers(ctx)+ if listErr != nil {+ containerChan <- containerMapResult{err: fmt.Errorf("failed to list containers: %w", listErr)}+ return+ }+ containerMap := make(map[string][]string)+ for _, c := range containers {+ for _, m := range c.Mounts {+ if m.Type == mount.TypeVolume && m.Name != "" {+ containerMap[m.Name] = append(containerMap[m.Name], c.ID)+ }+ }+ }+ containerChan <- containerMapResult{containerMap: containerMap}+ }()// Wait for both resultsvolResult := <-volChan@@ -954,7 +841,10 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat// background so this list request never waits for Docker's DiskUsage call.var usageVolumes []volume.Volumeif params.Sort == "size" {- if uv, found := docker.GetVolumeUsageDataStaleWhileRevalidate(apiCtx, dockerClient).Get(); found && (len(uv) > 0 || len(volResult.volumes) == 0) {+ usage, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (mo.Option[[]volume.Volume], error) {+ return docker.GetVolumeUsageDataStaleWhileRevalidate(ctx, dockerClient), nil+ })+ if uv, found := usage.Get(); found && (len(uv) > 0 || len(volResult.volumes) == 0) {usageVolumes = uvusageCacheSnapshot = "available"} else {@@ -964,12 +854,12 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat}}- enrichedVolumes := enrichVolumesWithUsageDataInternal(volResult.volumes, usageVolumes)+ enrichedVolumes := enrichVolumesWithUsageData(volResult.volumes, usageVolumes)items := make([]volumetypes.Volume, 0, len(enrichedVolumes))for _, v := range enrichedVolumes {volDto := volumetypes.NewSummary(v)- if !includeInternal && s.isInternalVolumeInternal(volDto) {+ if !includeInternal && s.isInternalVolume(volDto) {continue}if containerIDs, ok := volumeContainerMap[v.Name]; ok {@@ -981,13 +871,20 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginatitems = append(items, volDto)}- paginationConfig := s.buildVolumePaginationConfig()- result := paginationConfig.SearchOrderAndPaginate(items, effectiveParams)- counts := calculateVolumeUsageCountsInternal(items)+ result := s.buildVolumePaginationConfig().SearchOrderAndPaginate(items, effectiveParams)+ inUseCount := 0+ for _, v := range items {+ inUseCount += kit.Ternary(v.InUse, 1, 0)+ }+ counts := volumetypes.UsageCounts{+ Inuse: inUseCount,+ Unused: len(items) - inUseCount,+ Total: len(items),+ }paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, effectiveParams)slog.DebugContext(ctx, "volume service: listed volumes",- "dockerHost", dockerClient.DaemonHost(),+ "dockerHost", s.dockerService.DockerHost(),"requestedSort", params.Sort,"requestedOrder", params.Order,"effectiveSort", effectiveParams.Sort,@@ -1070,19 +967,21 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strindockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "connect", err)+ s.logVolumeRenameError(ctx, oldName, newName, user, "connect", err)return nil, fmt.Errorf("failed to connect to Docker: %w", err)}- source, err := dockerClient.VolumeInspect(ctx, oldName, client.VolumeInspectOptions{})+ source, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {+ return dockerClient.VolumeInspect(ctx, oldName, client.VolumeInspectOptions{})+ })if err != nil {if errdefs.IsNotFound(err) {err = common.Classify(common.ErrNotFound, err)}- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "inspect", err)+ s.logVolumeRenameError(ctx, oldName, newName, user, "inspect", err)return nil, fmt.Errorf("inspect source volume: %w", err)}- if s.isInternalVolumeInternal(volumetypes.NewSummary(source.Volume)) {+ if s.isInternalVolume(volumetypes.NewSummary(source.Volume)) {return nil, common.ErrVolumeRenameProtected}@@ -1092,23 +991,23 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strinslog.WarnContext(ctx, "could not stop volume browse helper before rename", "volume", oldName, "error", stopErr.Error())}- migration, err := volumes.PlanRename(ctx, dockerClient, oldName, newName, s.toolsImageInternal())+ migration, err := volumes.PlanRename(ctx, dockerClient, oldName, newName, s.toolsImage())if err != nil {- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "plan", err)+ s.logVolumeRenameError(ctx, oldName, newName, user, "plan", err)return nil, err}if applyErr := migration.Apply(ctx); applyErr != nil {if errdefs.IsInvalidArgument(applyErr) {applyErr = common.Classify(common.ErrBadRequest, applyErr)}- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "apply", applyErr)+ s.logVolumeRenameError(ctx, oldName, newName, user, "apply", applyErr)return nil, applyErr}- if renameVolumeMetadataErr := s.renameVolumeMetadataInternal(ctx, oldName, newName); renameVolumeMetadataErr != nil {+ if renameVolumeMetadataErr := s.renameVolumeMetadata(ctx, oldName, newName); renameVolumeMetadataErr != nil {rollbackErr := migration.Rollback(ctx)combinedErr := errors.Join(fmt.Errorf("rename volume metadata: %w", renameVolumeMetadataErr), rollbackErr)- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "metadata", combinedErr)+ s.logVolumeRenameError(ctx, oldName, newName, user, "metadata", combinedErr)return nil, combinedErr}@@ -1119,21 +1018,21 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strin// failed, so the rename itself succeeded.slog.WarnContext(ctx, "volume renamed but source volume could not be removed", "oldVolume", oldName, "newVolume", newName, "error", commitErr.Error())} else {- metadataErr := s.renameVolumeMetadataInternal(ctx, newName, oldName)+ metadataErr := s.renameVolumeMetadata(ctx, newName, oldName)rollbackErr := migration.Rollback(ctx)combinedErr := errors.Join(commitErr, metadataErr, rollbackErr)- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "commit", combinedErr)+ s.logVolumeRenameError(ctx, oldName, newName, user, "commit", combinedErr)return nil, combinedErr}}}- s.removeHelperEntry(oldName)+ s.takeHelperID(oldName)docker.InvalidateVolumeUsageCache(dockerClient)renamed, err := s.GetVolumeByName(ctx, newName)if err != nil {- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "inspect-renamed", err)+ s.logVolumeRenameError(ctx, oldName, newName, user, "inspect-renamed", err)return nil, fmt.Errorf("inspect renamed volume: %w", err)}@@ -1147,7 +1046,7 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strin// Backup jobs are keyed by policy ID and re-read the policy row on every run,// so renaming volume_name requires no job rescheduling.-func (s *VolumeService) renameVolumeMetadataInternal(ctx context.Context, oldName, newName string) error {+func (s *VolumeService) renameVolumeMetadata(ctx context.Context, oldName, newName string) error {if s.db == nil {return nil}@@ -1163,7 +1062,7 @@ func (s *VolumeService) renameVolumeMetadataInternal(ctx context.Context, oldNam})}-func (s *VolumeService) logVolumeRenameErrorInternal(ctx context.Context, oldName, newName string, user usertypes.Actor, step string, err error) {+func (s *VolumeService) logVolumeRenameError(ctx context.Context, oldName, newName string, user usertypes.Actor, step string, err error) {s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", oldName, oldName, user.ID, user.Username, "0", err, database.JSON{"action": "rename","step": step,diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.goindex f55e3543a..c3162c249 100644--- a/backend/internal/volume/service_test.go+++ b/backend/internal/volume/service_test.go@@ -39,7 +39,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx")-func TestEnrichVolumesWithUsageDataInternal(t *testing.T) {+func TestEnrichVolumesWithUsageData(t *testing.T) {tests := []struct {name stringvolumes []volume.Volume@@ -85,18 +85,18 @@ func TestEnrichVolumesWithUsageDataInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := enrichVolumesWithUsageDataInternal(tt.volumes, tt.usageVolumes)+ got := enrichVolumesWithUsageData(tt.volumes, tt.usageVolumes)require.Len(t, got, tt.wantLen)tt.assertions(t, got)})}}-func TestIsInternalVolumeInternal(t *testing.T) {+func TestIsInternalVolume(t *testing.T) {svc := &VolumeService{backupVolumeName: "arcane-backups"}- require.True(t, svc.isInternalVolumeInternal(volumetypes.Volume{Name: "arcane-backups"}))- require.False(t, svc.isInternalVolumeInternal(volumetypes.Volume{Name: "user-volume"}))+ require.True(t, svc.isInternalVolume(volumetypes.Volume{Name: "arcane-backups"}))+ require.False(t, svc.isInternalVolume(volumetypes.Volume{Name: "user-volume"}))}func TestRenameVolumeRejectsInvalidAndProtectedNames(t *testing.T) {@@ -112,7 +112,7 @@ func TestRenameVolumeRejectsInvalidAndProtectedNames(t *testing.T) {require.ErrorIs(t, err, common.ErrVolumeRenameProtected)}-func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {+func applyVolumeBackupMigrations(t *testing.T, gormDB *gorm.DB) {t.Helper()for _, name := range []string{"032_add_volume_backups.sql", "073_add_backup_support.sql", "087_add_volume_backup_remote_instance.sql"} {migration, err := os.ReadFile("../../resources/migrations/sqlite/" + name)@@ -121,15 +121,15 @@ func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {}}-func TestRenameVolumeMetadataInternalPreservesPoliciesAndHistory(t *testing.T) {+func TestRenameVolumeMetadataPreservesPoliciesAndHistory(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:volume-rename-metadata?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- applyVolumeBackupMigrationsInternal(t, gormDB)+ applyVolumeBackupMigrations(t, gormDB)require.NoError(t, gormDB.Exec("INSERT INTO volume_backup_policies (id, volume_name, schedule) VALUES (?, ?, ?)", "policy-1", "source-data", "0 0 2 * * *").Error)require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "source-data", PolicyID: "policy-1"}).Error)service := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil)- require.NoError(t, service.renameVolumeMetadataInternal(t.Context(), "source-data", "renamed-data"))+ require.NoError(t, service.renameVolumeMetadata(t.Context(), "source-data", "renamed-data"))var renamedPolicy stringrequire.NoError(t, gormDB.Raw("SELECT volume_name FROM volume_backup_policies WHERE id = ?", "policy-1").Scan(&renamedPolicy).Error)@@ -140,33 +140,7 @@ func TestRenameVolumeMetadataInternalPreservesPoliciesAndHistory(t *testing.T) {require.Equal(t, "renamed-data", backupEntry.VolumeName)}-func TestBuildVolumePruneOptionsInternal_PreservesInternalVolumes(t *testing.T) {- options := buildVolumePruneOptionsInternal(true)-- require.True(t, options.All)- require.NotNil(t, options.Filters)- require.True(t, options.Filters["label!"][internalVolumePruneFilterValue])-}--func TestBuildVolumePruneOptionsInternal_PreservesInternalVolumesForAnonymousVolumes(t *testing.T) {- options := buildVolumePruneOptionsInternal(false)-- require.False(t, options.All)- require.NotNil(t, options.Filters)- require.True(t, options.Filters["label!"][internalVolumePruneFilterValue])-}--func TestBuildVolumePruneMetadataInternal(t *testing.T) {- metadata := buildVolumePruneMetadataInternal(true, 2, 4096)-- require.Equal(t, "prune", metadata["action"])- require.Equal(t, true, metadata["all"])- require.Equal(t, 2, metadata["volumesDeleted"])- require.EqualValues(t, 4096, metadata["spaceReclaimed"])- require.Equal(t, internalVolumePruneFilterValue, metadata["internalVolumeFilterLabel"])-}--func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {+func TestListBackupsPaginatedByManagementType(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-management?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)require.NoError(t, gormDB.AutoMigrate(&VolumeBackup{}))@@ -179,7 +153,7 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {}service := &VolumeService{db: &database.DB{DB: gormDB}}- systemRows, total, err := service.listBackupsInternal(t.Context(), "app", pagination.QueryParams{+ systemRows, total, err := service.listBackups(t.Context(), "app", pagination.QueryParams{Filters: map[string]string{"type": "system"},})require.NoError(t, err)@@ -187,7 +161,7 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {require.Len(t, systemRows, 1)require.Equal(t, backup.ManagementTypeSystem, systemRows[0].Type)- volumeRows, total, err := service.listBackupsInternal(t.Context(), "app", pagination.QueryParams{+ volumeRows, total, err := service.listBackups(t.Context(), "app", pagination.QueryParams{Filters: map[string]string{"type": "volume"},})require.NoError(t, err)@@ -198,7 +172,7 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {}}-func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {+func newVolumeWorkspaceTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {t.Helper()dialer := &net.Dialer{}dockerClient, err := client.New(@@ -214,7 +188,7 @@ func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.Sreturn dockerClient}-func writeDockerExecAttachResponseInternal(t *testing.T, w http.ResponseWriter, stdout string) {+func writeDockerExecAttachResponse(t *testing.T, w http.ResponseWriter, stdout string) {t.Helper()hijacker, ok := w.(http.Hijacker)if !ok {@@ -274,7 +248,7 @@ func TestVolumeWorkspaceReadsWaitForMutationLock(t *testing.T) {}))t.Cleanup(server.Close)- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClient(t, server))service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)unlockMutation := service.workspaceLocks.Lock("workspace-volume")readStarted := make(chan struct{})@@ -344,7 +318,7 @@ func TestDownloadVolumeWorkspaceFileHoldsReadLockUntilClosed(t *testing.T) {return}_, _ = output.Write(content)- writeDockerExecAttachResponseInternal(t, w, output.String())+ writeDockerExecAttachResponse(t, w, output.String())case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/exec/read-exec/json"):w.Header().Set("Content-Type", "application/json")_ = json.NewEncoder(w).Encode(map[string]any{"Running": false, "ExitCode": 0})@@ -354,7 +328,7 @@ func TestDownloadVolumeWorkspaceFileHoldsReadLockUntilClosed(t *testing.T) {}))t.Cleanup(server.Close)- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClient(t, server))service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)service.helperByVolume["workspace-volume"] = &volumeHelper{id: "helper", lastUsedAt: time.Now(), protocol: types.ProtocolVersion}reader, size, err := service.workspace.DownloadVolumeWorkspaceFile(t.Context(), "workspace-volume", "file.txt")@@ -407,7 +381,7 @@ func TestUpdateVolumeWorkspaceRejectsStaleRevisionBeforeStaging(t *testing.T) {"{\"end\":true,\"version\":2}\n","{\"version\":\"0.2.0\",\"revision\":\"test\",\"buildTime\":\"test\",\"protocol\":2}\n",)- writeDockerExecAttachResponseInternal(t, w, output)+ writeDockerExecAttachResponse(t, w, output)case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/exec/workspace-exec-") && strings.HasSuffix(r.URL.Path, "/json"):w.Header().Set("Content-Type", "application/json")_ = json.NewEncoder(w).Encode(map[string]any{"Running": false, "ExitCode": 0})@@ -420,7 +394,7 @@ func TestUpdateVolumeWorkspaceRejectsStaleRevisionBeforeStaging(t *testing.T) {}))t.Cleanup(server.Close)- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)service := NewVolumeService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), nil, nil, nil, nil, nil, nil, nil, nil)uploadIndex := 0workspace, err := service.workspace.UpdateVolumeWorkspace(t.Context(), "workspace-volume", volumetypes.WorkspaceUpdateManifest{@@ -472,11 +446,11 @@ func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicyFromStore(t *testing.T) {gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-last-runs-store?mode=memory&cache=shared"), &gorm.Config{})require.NoError(t, err)- applyVolumeBackupMigrationsInternal(t, gormDB)+ applyVolumeBackupMigrations(t, gormDB)require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "first-policy", Status: VolumeBackupStatusSucceeded}).Error)require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "second-policy", Status: VolumeBackupStatusFailed}).Error)- store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStoreInternal()+ store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStore()first, err := store.Latest(t.Context(), "first-policy")require.NoError(t, err)require.Equal(t, "succeeded", first.Status)diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.goindex eb74b5ae6..140793645 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go@@ -325,7 +325,9 @@ func (s *Service) scanResult(// The name can move to another image after it was selected; never store that image's findings under imageID.// Compare layer diff IDs: image IDs differ by image store (containerd uses the manifest digest).if mode == "exec" {- inspect, inspectErr := dockerClient.ImageInspect(ctx, imageID)+ inspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {+ return cli.ImageInspect(ctx, imageID)+ })if inspectErr != nil {return nil, fmt.Errorf("selected image %s is no longer available: %w", imageID, inspectErr)}@@ -387,25 +389,31 @@ func (s *Service) Version(ctx context.Context) string {return ""}- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: &container.Config{- Image: scannerImage,- Entrypoint: []string{"trivy"},- Cmd: []string{"--version"},- Labels: map[string]string{- libarcane.InternalResourceLabel: "true",+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: &container.Config{+ Image: scannerImage,+ Entrypoint: []string{"trivy"},+ Cmd: []string{"--version"},+ Labels: map[string]string{+ libarcane.InternalResourceLabel: "true",+ },},- },- HostConfig: &container.HostConfig{- AutoRemove: true,- },+ HostConfig: &container.HostConfig{+ AutoRemove: true,+ },+ })})if err != nil {return ""}- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {+ return cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {+ return cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ })return ""}@@ -415,7 +423,9 @@ func (s *Service) Version(ctx context.Context) string {Follow: true,})if err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {+ return cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})+ })return ""}var closeLogsOnce sync.Once@@ -518,10 +528,6 @@ func (s *Service) resolveTrivyRuntimeOptions(ctx context.Context, dockerClient *}, nil}-func (s *Service) scanTimeoutArg() string {- return timeouts.GetDuration(s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()-}-func (s *Service) EnsureImage(ctx context.Context) (string, error) {dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {@@ -529,7 +535,9 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {}scannerImage := volumehelper.ToolsImage(s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value)- if _, imageInspectErr := dockerClient.ImageInspect(ctx, scannerImage); imageInspectErr == nil {+ if _, imageInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {+ return cli.ImageInspect(ctx, scannerImage)+ }); imageInspectErr == nil {return scannerImage, nil}@@ -553,20 +561,19 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {}func (s *Service) ensureTrivyCacheVolume(ctx context.Context) (string, error) {- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return "", err- }-- if _, volumeInspectErr := dockerClient.VolumeInspect(ctx, trivyCacheVolumeName, client.VolumeInspectOptions{}); volumeInspectErr == nil {+ if _, volumeInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.VolumeInspectResult, error) {+ return cli.VolumeInspect(ctx, trivyCacheVolumeName, client.VolumeInspectOptions{})+ }); volumeInspectErr == nil {return trivyCacheVolumeName, nil}- _, err = dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{- Name: trivyCacheVolumeName,- Labels: map[string]string{- libarcane.InternalResourceLabel: "true",- },+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.VolumeCreateResult, error) {+ return cli.VolumeCreate(ctx, client.VolumeCreateOptions{+ Name: trivyCacheVolumeName,+ Labels: map[string]string{+ libarcane.InternalResourceLabel: "true",+ },+ })})if err != nil {return "", fmt.Errorf("failed to create trivy cache volume: %w", err)@@ -614,8 +621,10 @@ func (s *Service) trivyContainerFiles(ctx context.Context) map[string][]byte {// Flags passed here take precedence over the user-supplied trivy.yaml.func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string, files map[string][]byte) []string {serverURL, _ := s.ServerConfig()- cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.scanTimeoutArg()}- cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, s.settingsService.GetSettingsConfig().TrivyDbRegistry.Value)...)+ cfg := s.settingsService.GetSettingsConfig()+ scanTimeout := timeouts.GetDuration(cfg.TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()+ cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", scanTimeout}+ cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, cfg.TrivyDbRegistry.Value)...)cmdArgs = append(cmdArgs, vuln.IgnoreUnfixedArgs(s.IgnoreUnfixed())...)if files[vuln.TrivyConfigFile] != nil {@@ -646,12 +655,11 @@ func (s *Service) createAndStartTrivyContainer() (string, error) {vuln.LogContainerStartupRequest(ctx, scope, containerConfig, hostConfig)- createCtx, createCancel := context.WithTimeout(ctx, timeouts.GetDuration(0, timeouts.DefaultDockerAPI))- defer createCancel()-- resp, err := dockerClient.ContainerCreate(createCtx, client.ContainerCreateOptions{- Config: containerConfig,- HostConfig: hostConfig,+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{+ Config: containerConfig,+ HostConfig: hostConfig,+ })})if err != nil {slog.WarnContext(ctx, "failed to create trivy container", "scope", scope, "error", err)@@ -672,10 +680,9 @@ func (s *Service) createAndStartTrivyContainer()}- startCtx, startCancel := context.WithTimeout(ctx, timeouts.GetDuration(0, timeouts.DefaultDockerAPI))- defer startCancel()-- if _, containerStartErr := dockerClient.ContainerStart(startCtx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {+ return cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})+ }); containerStartErr != nil {stdoutLog, stderrLog := vuln.ReadStartupLogs(ctx, dockerClient, resp.ID)slog.WarnContext(ctx,"failed to start trivy container",@@ -902,7 +909,7 @@ func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID strlogDone := docker.StartStdCopy(logs, stdoutFile, stderrFile)startTime := time.Now()- waitCtx, waitCancel := context.WithTimeout(ctx, vuln.ContainerWaitTimeout(s.scanTimeoutArg()))+ waitCtx, waitCancel := context.WithTimeout(ctx, vuln.ContainerWaitTimeout(timeouts.GetDuration(s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()))defer waitCancel()waitResult := dockerClient.ContainerWait(waitCtx, containerID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})diff --git a/backend/internal/vulnerability/children/scan/service.go b/backend/internal/vulnerability/children/scan/service.goindex 3b5b92242..06030c735 100644--- a/backend/internal/vulnerability/children/scan/service.go+++ b/backend/internal/vulnerability/children/scan/service.go@@ -31,7 +31,6 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan/children/trivy"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/utils")@@ -176,11 +175,9 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user use}scanCtx := utils.ActivityRuntimeContext(ctx, nil)- dockerClient, err := s.dockerService.GetClient(ctx)- if err != nil {- return nil, fmt.Errorf("failed to connect to Docker: %w", err)- }- imageInspect, err := dockerClient.ImageInspect(scanCtx, imageID)+ imageInspect, err := s.dockerService.Call(scanCtx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {+ return cli.ImageInspect(ctx, imageID)+ })if err != nil {return nil, fmt.Errorf("failed to inspect image: %w", err)}@@ -556,12 +553,10 @@ func (s *Service) ScanTarget(ctx context.Context, envID string, target Target) (defer func() {// A cancel only detaches from the exec, so the container is restarted to stop Trivy before another scan reuses it.if ctx.Err() != nil {- restartCtx, cancelRestart := context.WithTimeout(context.WithoutCancel(ctx), timeouts.GetDuration(0, timeouts.DefaultDockerAPI))- defer cancelRestart()- dockerClient, clientErr := s.dockerService.GetClient(restartCtx)- if clientErr == nil {- _, clientErr = dockerClient.ContainerRestart(restartCtx, containerID, client.ContainerRestartOptions{})- }+ restartCtx := context.WithoutCancel(ctx)+ _, clientErr := s.dockerService.Call(restartCtx, func(ctx context.Context, cli *client.Client) (client.ContainerRestartResult, error) {+ return cli.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})+ })if clientErr != nil {// Without this container the pool cannot serve scans, so later scans start a fresh one.slog.WarnContext(restartCtx, "scheduled vulnerability scan: failed to stop a canceled scan; retiring its scan containers", "containerId", containerID, "error", clientErr)diff --git a/backend/internal/vulnerability/children/store/service.go b/backend/internal/vulnerability/children/store/service.goindex 7a2d961ca..59374fe8c 100644--- a/backend/internal/vulnerability/children/store/service.go+++ b/backend/internal/vulnerability/children/store/service.go@@ -213,12 +213,6 @@ func (s *Service) ImportLegacyReportFiles(ctx context.Context) {}}-// DeleteScanResult deletes the scan result for an image-func (s *Service) DeleteScanResult(ctx context.Context, imageID string) error {- s.deleteScanChildRows(ctx, []string{imageID})- return s.records.Delete(ctx, []string{imageID})-}-// DeleteScanResultsByImageIDs deletes scan results for multiple images in one query.func (s *Service) DeleteScanResultsByImageIDs(ctx context.Context, imageIDs []string) error {if len(imageIDs) == 0 {diff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.goindex 34d0eac4d..13ce3b9bf 100644--- a/backend/internal/vulnerability/service.go+++ b/backend/internal/vulnerability/service.go@@ -450,11 +450,6 @@ func (s *VulnerabilityService) SyncThreatIntel(ctx context.Context, force bool)return s.store.SyncThreatIntel(ctx, force)}-// DeleteScanResult deletes the scan result for an image.-func (s *VulnerabilityService) DeleteScanResult(ctx context.Context, imageID string) error {- return s.store.DeleteScanResult(ctx, imageID)-}-// DeleteScanResultsByImageIDs deletes scan results for multiple images.func (s *VulnerabilityService) DeleteScanResultsByImageIDs(ctx context.Context, imageIDs []string) error {return s.store.DeleteScanResultsByImageIDs(ctx, imageIDs)diff --git a/backend/pkg/libarcane/startup/runtime_identity.go b/backend/pkg/libarcane/startup/runtime_identity.goindex 8fd607e2d..822d217e1 100644--- a/backend/pkg/libarcane/startup/runtime_identity.go+++ b/backend/pkg/libarcane/startup/runtime_identity.go@@ -20,7 +20,7 @@ import (const (defaultDataDirectory = "/app/data"defaultBuildsDirectory = "/builds"- defaultDatabaseURL = "file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate"+ defaultDatabaseURL = "file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate"defaultDockerConfigDir = "/app/data/.docker"defaultDockerSocketPath = "/var/run/docker.sock"defaultRuntimeUID = 65532@@ -66,8 +66,8 @@ func ApplyRequestedRuntimeIdentity(ctx context.Context, cfg *RuntimeIdentityConfprojectsDir = filepath.Clean(strings.TrimSpace(cfg.ProjectsDirectory))}- inContainer := runningInContainerInternal(os.Getenv, os.Stat)- req, warning, err := loadRuntimeIdentityRequestInternal(cfg, inContainer)+ inContainer := runningInContainer(os.Getenv, os.Stat)+ req, warning, err := loadRuntimeIdentityRequest(cfg, inContainer)if warning != "" {slog.WarnContext(ctx, "Runtime identity warning", "warning", warning)}@@ -75,50 +75,37 @@ func ApplyRequestedRuntimeIdentity(ctx context.Context, cfg *RuntimeIdentityConfreturn err}if !req.Enabled {- return ensureSQLiteFilesExistInternal(cfg.DatabaseURL)+ return ensureSQLiteFilesExist(cfg.DatabaseURL)}- runtimeUID := req.UID- runtimeGID := req.GID-- // Avoid re-execing forever when the requested runtime identity is already active.- if os.Geteuid() == runtimeUID && os.Getegid() == runtimeGID {- if ensureRuntimeDockerConfigErr := ensureRuntimeDockerConfigInternal(cfg, os.Setenv, runtimeUID, runtimeGID, inContainer); ensureRuntimeDockerConfigErr != nil {- return ensureRuntimeDockerConfigErr- }- return ensureSQLiteFilesExistInternal(cfg.DatabaseURL)+ if configErr := ensureRuntimeDockerConfig(cfg, os.Setenv, req.UID, req.GID, inContainer); configErr != nil {+ return configErr}- if os.Geteuid() != 0 {+ switch {+ case os.Geteuid() == req.UID && os.Getegid() == req.GID:+ // Avoid re-execing forever when the requested runtime identity is already active.+ return ensureSQLiteFilesExist(cfg.DatabaseURL)+ case os.Geteuid() != 0:slog.WarnContext(ctx, "Runtime identity warning: process is not root, continuing as current user",- "euid", os.Geteuid(), "puid", runtimeUID, "pgid", runtimeGID)- if ensureRuntimeDockerConfigErr2 := ensureRuntimeDockerConfigInternal(cfg, os.Setenv, runtimeUID, runtimeGID, inContainer); ensureRuntimeDockerConfigErr2 != nil {- return ensureRuntimeDockerConfigErr2- }- return ensureSQLiteFilesExistInternal(cfg.DatabaseURL)- }-- if ensureRuntimeDockerConfigErr3 := ensureRuntimeDockerConfigInternal(cfg, os.Setenv, runtimeUID, runtimeGID, inContainer); ensureRuntimeDockerConfigErr3 != nil {- return ensureRuntimeDockerConfigErr3+ "euid", os.Geteuid(), "puid", req.UID, "pgid", req.GID)+ return ensureSQLiteFilesExist(cfg.DatabaseURL)}if inContainer {- mountpoints, loadMountpointsErr := loadMountpointsInternal(mountInfoPath)- if loadMountpointsErr != nil {- return fmt.Errorf("load mountpoints: %w", loadMountpointsErr)+ mountInfo, readErr := os.ReadFile(mountInfoPath)+ if readErr != nil && !os.IsNotExist(readErr) {+ return fmt.Errorf("load mountpoints: %w", readErr)}-- if prepareWritablePathsWithRootsErr := prepareWritablePathsWithRootsInternal(- runtimeUID, runtimeGID, mountpoints, projectsDir, defaultDataDirectory, defaultBuildsDirectory,- ); prepareWritablePathsWithRootsErr != nil {- return prepareWritablePathsWithRootsErr+ if prepareErr := prepareWritablePathsWithRoots(req.UID, req.GID, parseMountpoints(string(mountInfo)), projectsDir, defaultDataDirectory, defaultBuildsDirectory); prepareErr != nil {+ return prepareErr}}- return reexecWithRuntimeIdentityInternal(ctx, req)+ return reexecWithRuntimeIdentity(ctx, req)}-func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainer bool) (runtimeIdentityRequest, string, error) {+func loadRuntimeIdentityRequest(cfg *RuntimeIdentityConfig, inContainer bool) (runtimeIdentityRequest, string, error) {if cfg == nil {cfg = &RuntimeIdentityConfig{}}@@ -126,12 +113,22 @@ func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainerpuid := strings.TrimSpace(cfg.PUID)pgid := strings.TrimSpace(cfg.PGID)- if puid == "" && pgid == "" {- return defaultRuntimeIdentityRequestInternal(cfg.DockerHost, inContainer), "", nil- }-if puid == "" || pgid == "" {- req := defaultRuntimeIdentityRequestInternal(cfg.DockerHost, inContainer)+ // Without both overrides, containers run as the default non-root user.+ req := runtimeIdentityRequest{DockerHost: cfg.DockerHost}+ if inContainer {+ req = runtimeIdentityRequest{+ Enabled: true,+ UID: defaultRuntimeUID,+ GID: defaultRuntimeGID,+ CredentialUID: uint32(defaultRuntimeUID),+ CredentialGID: uint32(defaultRuntimeGID),+ DockerHost: cfg.DockerHost,+ }+ }+ if puid == "" && pgid == "" {+ return req, "", nil+ }warning := kit.Ternary(inContainer,"PUID and PGID must both be set to override the default non-root runtime user; continuing with the default non-root runtime user",@@ -140,12 +137,12 @@ func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainerreturn req, warning, nil}- uid, credentialUID, err := parseRuntimeIdentityValueInternal(puid, "PUID")+ uid, credentialUID, err := parseRuntimeIdentityValue(puid, "PUID")if err != nil {return runtimeIdentityRequest{}, "", fmt.Errorf("invalid PUID %q: %w", puid, err)}- gid, credentialGID, err := parseRuntimeIdentityValueInternal(pgid, "PGID")+ gid, credentialGID, err := parseRuntimeIdentityValue(pgid, "PGID")if err != nil {return runtimeIdentityRequest{}, "", fmt.Errorf("invalid PGID %q: %w", pgid, err)}@@ -160,25 +157,7 @@ func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainer}, "", nil}-func defaultRuntimeIdentityRequestInternal(dockerHost string, inContainer bool) runtimeIdentityRequest {- if !inContainer {- return runtimeIdentityRequest{- Enabled: false,- DockerHost: dockerHost,- }- }-- return runtimeIdentityRequest{- Enabled: true,- UID: defaultRuntimeUID,- GID: defaultRuntimeGID,- CredentialUID: uint32(defaultRuntimeUID),- CredentialGID: uint32(defaultRuntimeGID),- DockerHost: dockerHost,- }-}--func runningInContainerInternal(getenv func(string) string, stat func(string) (os.FileInfo, error)) bool {+func runningInContainer(getenv func(string) string, stat func(string) (os.FileInfo, error)) bool {if kit.ParseOrDefault(strings.TrimSpace(getenv("ARCANE_IN_CONTAINER")), false, strconv.ParseBool) {return true}@@ -196,8 +175,8 @@ func runningInContainerInternal(getenv func(string) string, stat func(string) (oreturn false}-func ensureRuntimeDockerConfigInternal(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) error {- configDir, err := configureRuntimeDockerConfigEnvInternal(cfg, setenv, uid, gid, inContainer)+func ensureRuntimeDockerConfig(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) error {+ configDir, err := configureRuntimeDockerConfigEnv(cfg, setenv, uid, gid, inContainer)if err != nil {return err}@@ -218,7 +197,7 @@ func ensureRuntimeDockerConfigInternal(cfg *RuntimeIdentityConfig, setenv func(sreturn nil}-func configureRuntimeDockerConfigEnvInternal(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) (string, error) {+func configureRuntimeDockerConfigEnv(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) (string, error) {if cfg == nil {cfg = &RuntimeIdentityConfig{}}@@ -246,51 +225,28 @@ func configureRuntimeDockerConfigEnvInternal(cfg *RuntimeIdentityConfig, setenvreturn configDir, nil}-func runtimeIdentitySupplementaryGroupsInternal(dockerHost string, resolveSocketGroup func(string) mo.Option[uint32]) []uint32 {- socketPath, ok := dockerSocketPathInternal(dockerHost).Get()- if !ok {- return nil+func runtimeIdentitySupplementaryGroups(dockerHost string, socketGroup func(string) mo.Option[uint32]) []uint32 {+ socketPath := defaultDockerSocketPath+ if value := strings.TrimSpace(dockerHost); value != "" {+ parsed, err := url.Parse(value)+ if err != nil || parsed.Scheme != "unix" {+ return nil+ }+ socketPath = cmp.Or(parsed.Host+parsed.Path, parsed.Opaque)+ if socketPath == "" {+ return nil+ }+ socketPath = filepath.Clean("/" + strings.TrimPrefix(socketPath, "/"))}- socketGID, ok := resolveSocketGroup(socketPath).Get()+ socketGID, ok := socketGroup(socketPath).Get()if !ok {return nil}return []uint32{socketGID}}-func dockerSocketPathInternal(raw string) mo.Option[string] {- value := strings.TrimSpace(raw)- if value == "" {- return mo.Some(defaultDockerSocketPath)- }-- parsed, err := url.Parse(value)- if err != nil || parsed.Scheme != "unix" {- return mo.None[string]()- }-- if parsed.Host != "" || parsed.Path != "" {- socketPath := parsed.Host + parsed.Path- if !strings.HasPrefix(socketPath, "/") {- socketPath = "/" + socketPath- }- return mo.Some(filepath.Clean(socketPath))- }-- if parsed.Opaque == "" {- return mo.None[string]()- }-- socketPath := strings.TrimPrefix(parsed.Opaque, "//")- if !strings.HasPrefix(socketPath, "/") {- socketPath = "/" + socketPath- }-- return mo.Some(filepath.Clean(socketPath))-}--func prepareWritablePathsWithRootsInternal(uid, gid int, mountpoints map[string]struct{}, projectsDir, dataDirectory, buildsDirectory string) error {+func prepareWritablePathsWithRoots(uid, gid int, mountpoints map[string]struct{}, projectsDir, dataDirectory, buildsDirectory string) error {if err := os.MkdirAll(dataDirectory, utils.DirPerm); err != nil {return fmt.Errorf("create data directory: %w", err)}@@ -318,7 +274,7 @@ func prepareWritablePathsWithRootsInternal(uid, gid int, mountpoints map[string]}continue}- if chownRecursiveErr := chownRecursiveInternal(entryPath, uid, gid, mountpoints, projectsDir); chownRecursiveErr != nil {+ if chownRecursiveErr := chownRecursive(entryPath, uid, gid, mountpoints, projectsDir); chownRecursiveErr != nil {return fmt.Errorf("chown %s: %w", entryPath, chownRecursiveErr)}}@@ -337,15 +293,15 @@ func prepareWritablePathsWithRootsInternal(uid, gid int, mountpoints map[string]return fmt.Errorf("stat builds directory: %w", statErr)}- if chownRecursiveErr2 := chownRecursiveInternal(buildsDirectory, uid, gid, mountpoints, projectsDir); chownRecursiveErr2 != nil {+ if chownRecursiveErr2 := chownRecursive(buildsDirectory, uid, gid, mountpoints, projectsDir); chownRecursiveErr2 != nil {return fmt.Errorf("chown builds directory: %w", chownRecursiveErr2)}return nil}-func ensureSQLiteFilesExistInternal(databaseURL string) error {- sqlitePath, ok, err := sqliteDatabasePathInternal(databaseURL)+func ensureSQLiteFilesExist(databaseURL string) error {+ sqlitePath, ok, err := sqliteDatabasePath(databaseURL)if err != nil {return err}@@ -355,7 +311,7 @@ func ensureSQLiteFilesExistInternal(databaseURL string) error {// Ensure the parent directory exists before creating the file.// This covers the "already the right user" early-return path where- // prepareWritablePathsWithRootsInternal is not called.+ // prepareWritablePathsWithRoots is not called.dir := filepath.Dir(sqlitePath)if dir != "" && dir != "." {if mkdirAllErr := os.MkdirAll(dir, utils.DirPerm); mkdirAllErr != nil {@@ -374,28 +330,16 @@ func ensureSQLiteFilesExistInternal(databaseURL string) error {return nil}-func sqliteDatabasePathInternal(databaseURL string) (string, bool, error) {+func sqliteDatabasePath(databaseURL string) (string, bool, error) {value := cmp.Or(strings.TrimSpace(databaseURL), defaultDatabaseURL)if !strings.HasPrefix(value, "file:") {return "", false, nil}- parsed, err := url.Parse(value)+ pathPart, err := kit.SQLitePathFromDSN(value)if err != nil {return "", false, fmt.Errorf("parse sqlite database url: %w", err)}-- // For relative URLs like "file:data/arcane.db", url.Parse puts the path in- // Opaque (without a leading slash). For absolute URLs like "file:/app/data/arcane.db",- // Opaque is empty and Path contains the absolute path. Only strip the leading- // slash from the opaque portion to preserve absolute paths.- var pathPart string- if parsed.Opaque != "" {- pathPart = strings.TrimPrefix(parsed.Opaque, "/")- } else {- pathPart = parsed.Path- }-if pathPart == "" || strings.HasPrefix(pathPart, ":memory:") {return "", false, nil}@@ -403,7 +347,7 @@ func sqliteDatabasePathInternal(databaseURL string) (string, bool, error) {return filepath.Clean(pathPart), true, nil}-func chownRecursiveInternal(path string, uid, gid int, mountpoints map[string]struct{}, projectsDir string) error {+func chownRecursive(path string, uid, gid int, mountpoints map[string]struct{}, projectsDir string) error {return filepath.Walk(path, func(currentPath string, info os.FileInfo, err error) error {if err != nil {return err@@ -425,18 +369,7 @@ func chownRecursiveInternal(path string, uid, gid int, mountpoints map[string]st})}-func loadMountpointsInternal(path string) (map[string]struct{}, error) {- data, err := os.ReadFile(path)- if err != nil {- if os.IsNotExist(err) {- return map[string]struct{}{}, nil- }- return nil, err- }- return parseMountpointsInternal(string(data)), nil-}--func parseMountpointsInternal(data string) map[string]struct{} {+func parseMountpoints(data string) map[string]struct{} {mountpoints := make(map[string]struct{})for line := range strings.SplitSeq(data, "\n") {@@ -450,17 +383,17 @@ func parseMountpointsInternal(data string) map[string]struct{} {continue}- mountpoint := filepath.Clean(unescapeMountInfoPathInternal(fields[4]))+ mountpoint := filepath.Clean(unescapeMountInfoPath(fields[4]))mountpoints[mountpoint] = struct{}{}}return mountpoints}-// unescapeMountInfoPathInternal decodes the kernel's octal escape sequences+// unescapeMountInfoPath decodes the kernel's octal escape sequences// used in /proc/self/mountinfo. The kernel only uses \040 (space), \011 (tab),// \012 (newline), and \134 (backslash) — no other escape forms appear.-func unescapeMountInfoPathInternal(path string) string {+func unescapeMountInfoPath(path string) string {replacer := strings.NewReplacer(`\040`, " ",`\011`, "\t",diff --git a/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go b/backend/pkg/libarcane/startup/runtime_identity_int_32bit.goindex 8ca6eed95..c0b68628c 100644--- a/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go+++ b/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go@@ -7,7 +7,7 @@ import ("strconv")-func parseRuntimeIdentityValueInternal(raw, key string) (int, uint32, error) {+func parseRuntimeIdentityValue(raw, key string) (int, uint32, error) {value, err := strconv.ParseInt(raw, 10, 32)if err != nil {return 0, 0, fmt.Errorf("parse %s: %w", key, err)diff --git a/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go b/backend/pkg/libarcane/startup/runtime_identity_int_64bit.goindex 9bc3015e3..c748d1ba4 100644--- a/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go+++ b/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go@@ -8,7 +8,7 @@ import ("strconv")-func parseRuntimeIdentityValueInternal(raw, key string) (int, uint32, error) {+func parseRuntimeIdentityValue(raw, key string) (int, uint32, error) {value, err := strconv.Atoi(raw)if err != nil {return 0, 0, fmt.Errorf("parse %s: %w", key, err)diff --git a/backend/pkg/libarcane/startup/runtime_identity_linux.go b/backend/pkg/libarcane/startup/runtime_identity_linux.goindex 87a6161f3..47c05648b 100644--- a/backend/pkg/libarcane/startup/runtime_identity_linux.go+++ b/backend/pkg/libarcane/startup/runtime_identity_linux.go@@ -13,13 +13,13 @@ import ("github.com/samber/mo")-func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityRequest) error {+func reexecWithRuntimeIdentity(ctx context.Context, req runtimeIdentityRequest) error {executable, err := os.Executable()if err != nil {return fmt.Errorf("resolve executable: %w", err)}- groups := runtimeIdentitySupplementaryGroupsInternal(req.DockerHost, resolveSocketGroupInternal)+ groups := runtimeIdentitySupplementaryGroups(req.DockerHost, resolveSocketGroup)cmd := exec.CommandContext(ctx, executable, os.Args[1:]...) //nolint:gosec // re-executing our own binary with the same args under a different UID/GID// Forward shutdown instead of the default SIGKILL so the child can unregister its actor host.@@ -59,7 +59,7 @@ func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityRreturn fmt.Errorf("wait for runtime identity child: %w", waitErr)}-func resolveSocketGroupInternal(socketPath string) mo.Option[uint32] {+func resolveSocketGroup(socketPath string) mo.Option[uint32] {// os.* rather than acfs: the docker socket is a host-configured system path// probed during identity bootstrap, not under any acfs root.info, err := os.Stat(socketPath)diff --git a/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go b/backend/pkg/libarcane/startup/runtime_identity_nonlinux.goindex 2146784c7..97f54e094 100644--- a/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go+++ b/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go@@ -8,11 +8,8 @@ import ("runtime")-var (- _ = runtimeIdentitySupplementaryGroupsInternal- _ = dockerSocketPathInternal-)+var _ = runtimeIdentitySupplementaryGroups-func reexecWithRuntimeIdentityInternal(_ context.Context, req runtimeIdentityRequest) error {+func reexecWithRuntimeIdentity(_ context.Context, req runtimeIdentityRequest) error {return fmt.Errorf("runtime identity switching is only supported on linux, current platform is %s", runtime.GOOS)}diff --git a/backend/pkg/libarcane/startup/runtime_identity_test.go b/backend/pkg/libarcane/startup/runtime_identity_test.goindex 6e3846294..cc8973631 100644--- a/backend/pkg/libarcane/startup/runtime_identity_test.go+++ b/backend/pkg/libarcane/startup/runtime_identity_test.go@@ -14,7 +14,7 @@ import (func TestLoadRuntimeIdentityRequest(t *testing.T) {t.Run("disabled when unset outside a container", func(t *testing.T) {- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{DockerHost: "unix:///tmp/docker.sock",}, false)require.NoError(t, err)@@ -24,7 +24,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {})t.Run("enabled with default non-root runtime when unset in container", func(t *testing.T) {- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{DockerHost: "unix:///tmp/docker.sock",}, true)require.NoError(t, err)@@ -38,7 +38,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {})t.Run("warning when partial config", func(t *testing.T) {- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "1001"}, true)+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "1001"}, true)require.NoError(t, err)require.Contains(t, warning, "PUID and PGID must both be set")require.True(t, req.Enabled)@@ -49,7 +49,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {})t.Run("warning when partial config outside container", func(t *testing.T) {- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "1001"}, false)+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "1001"}, false)require.NoError(t, err)require.Contains(t, warning, "PUID and PGID must both be set to enable runtime identity outside containers")require.False(t, req.Enabled)@@ -60,7 +60,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {})t.Run("error when invalid numeric value", func(t *testing.T) {- _, _, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "abc", PGID: "1001"}, false)+ _, _, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "abc", PGID: "1001"}, false)require.Error(t, err)require.Contains(t, err.Error(), "invalid PUID")})@@ -68,13 +68,13 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {t.Run("error when value exceeds uint32", func(t *testing.T) {tooLarge := strconv.FormatUint(uint64(math.MaxUint32)+1, 10)- _, _, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: tooLarge, PGID: "1001"}, false)+ _, _, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: tooLarge, PGID: "1001"}, false)require.Error(t, err)require.Contains(t, err.Error(), "invalid PUID")})t.Run("enabled when both set", func(t *testing.T) {- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "1001",PGID: "2001",DockerHost: "unix:///tmp/docker.sock",@@ -90,15 +90,15 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {})t.Run("error when value is negative", func(t *testing.T) {- _, _, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "-1", PGID: "1001"}, false)+ _, _, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "-1", PGID: "1001"}, false)require.Error(t, err)require.Contains(t, err.Error(), "invalid PUID")})}-func TestRunningInContainerInternal(t *testing.T) {+func TestRunningInContainer(t *testing.T) {t.Run("explicit arcane env enables container mode", func(t *testing.T) {- inContainer := runningInContainerInternal(+ inContainer := runningInContainer(func(key string) string {return kit.Ternary(key == "ARCANE_IN_CONTAINER", "true", "")},@@ -111,7 +111,7 @@ func TestRunningInContainerInternal(t *testing.T) {})t.Run("container env enables container mode", func(t *testing.T) {- inContainer := runningInContainerInternal(+ inContainer := runningInContainer(func(key string) string {return kit.Ternary(key == "container", "podman", "")},@@ -124,7 +124,7 @@ func TestRunningInContainerInternal(t *testing.T) {})t.Run("container marker file enables container mode", func(t *testing.T) {- inContainer := runningInContainerInternal(+ inContainer := runningInContainer(func(string) string { return "" },func(path string) (os.FileInfo, error) {return nil, kit.Ternary(path == "/run/.containerenv", nil, os.ErrNotExist)@@ -135,7 +135,7 @@ func TestRunningInContainerInternal(t *testing.T) {})t.Run("no signal disables container mode", func(t *testing.T) {- inContainer := runningInContainerInternal(+ inContainer := runningInContainer(func(string) string { return "" },func(string) (os.FileInfo, error) {return nil, os.ErrNotExist@@ -151,7 +151,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {cfg := &RuntimeIdentityConfig{}env := map[string]string{}- configDir, err := configureRuntimeDockerConfigEnvInternal(+ configDir, err := configureRuntimeDockerConfigEnv(cfg,func(key, value string) error {env[key] = value@@ -173,7 +173,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {env := map[string]string{}setCalled := false- configDir, err := configureRuntimeDockerConfigEnvInternal(+ configDir, err := configureRuntimeDockerConfigEnv(cfg,func(key, value string) error {setCalled = true@@ -197,7 +197,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {env := map[string]string{}setCalled := false- configDir, err := configureRuntimeDockerConfigEnvInternal(+ configDir, err := configureRuntimeDockerConfigEnv(cfg,func(key, value string) error {setCalled = true@@ -221,7 +221,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {env := map[string]string{}setCalled := false- configDir, err := configureRuntimeDockerConfigEnvInternal(+ configDir, err := configureRuntimeDockerConfigEnv(cfg,func(key, value string) error {setCalled = true@@ -244,7 +244,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {cfg := &RuntimeIdentityConfig{}env := map[string]string{}- configDir, err := configureRuntimeDockerConfigEnvInternal(+ configDir, err := configureRuntimeDockerConfigEnv(cfg,func(key, value string) error {env[key] = value@@ -266,7 +266,7 @@ func TestEnsureRuntimeDockerConfigCreatesCustomDirectory(t *testing.T) {customDir := filepath.Join(t.TempDir(), "custom-docker-config")cfg := &RuntimeIdentityConfig{DockerConfig: customDir}- require.NoError(t, ensureRuntimeDockerConfigInternal(+ require.NoError(t, ensureRuntimeDockerConfig(cfg,func(string, string) error {require.FailNow(t, "setenv should not be called for explicit DOCKER_CONFIG")@@ -285,7 +285,7 @@ func TestEnsureRuntimeDockerConfigCreatesCustomDirectory(t *testing.T) {func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {t.Run("maps default docker socket group when docker host unset", func(t *testing.T) {- groups := runtimeIdentitySupplementaryGroupsInternal(+ groups := runtimeIdentitySupplementaryGroups("",func(socketPath string) mo.Option[uint32] {require.Equal(t, defaultDockerSocketPath, socketPath)@@ -297,7 +297,7 @@ func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {})t.Run("maps custom unix docker host socket group", func(t *testing.T) {- groups := runtimeIdentitySupplementaryGroupsInternal(+ groups := runtimeIdentitySupplementaryGroups("unix:///tmp/docker.sock",func(socketPath string) mo.Option[uint32] {require.Equal(t, "/tmp/docker.sock", socketPath)@@ -311,7 +311,7 @@ func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {t.Run("skips non unix docker host", func(t *testing.T) {called := false- groups := runtimeIdentitySupplementaryGroupsInternal(+ groups := runtimeIdentitySupplementaryGroups("tcp://docker:2375",func(string) mo.Option[uint32] {called = true@@ -324,7 +324,7 @@ func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {})t.Run("skips socket group when socket lookup fails", func(t *testing.T) {- groups := runtimeIdentitySupplementaryGroupsInternal(+ groups := runtimeIdentitySupplementaryGroups("unix:///tmp/missing.sock",func(string) mo.Option[uint32] { return mo.None[uint32]() },)@@ -340,7 +340,7 @@ func TestParseMountpoints(t *testing.T) {99 92 0:46 / /builds rw,relatime - ext4 /dev/sdc1 rw`- parsed := parseMountpointsInternal(data)+ parsed := parseMountpoints(data)require.Contains(t, parsed, "/app/data")require.Contains(t, parsed, "/app/data/projects")require.Contains(t, parsed, "/builds")@@ -348,39 +348,39 @@ func TestParseMountpoints(t *testing.T) {func TestSQLiteDatabasePath(t *testing.T) {t.Run("uses default sqlite path when unset", func(t *testing.T) {- path, ok, err := sqliteDatabasePathInternal("")+ path, ok, err := sqliteDatabasePath("")require.NoError(t, err)require.True(t, ok)require.Equal(t, "data/arcane.db", path)})t.Run("preserves absolute sqlite path", func(t *testing.T) {- path, ok, err := sqliteDatabasePathInternal("file:/app/custom/arcane.db?_pragma=journal_mode(WAL)")+ path, ok, err := sqliteDatabasePath("file:/app/custom/arcane.db?_pragma=journal_mode(WAL)")require.NoError(t, err)require.True(t, ok)require.Equal(t, "/app/custom/arcane.db", path)})t.Run("returns relative sqlite path", func(t *testing.T) {- path, ok, err := sqliteDatabasePathInternal("file:data/arcane.db?_pragma=journal_mode(WAL)")+ path, ok, err := sqliteDatabasePath("file:data/arcane.db?_pragma=journal_mode(WAL)")require.NoError(t, err)require.True(t, ok)require.Equal(t, "data/arcane.db", path)})t.Run("skips non sqlite database urls", func(t *testing.T) {- path, ok, err := sqliteDatabasePathInternal("postgres://arcane:secret@db/arcane")+ path, ok, err := sqliteDatabasePath("postgres://arcane:secret@db/arcane")require.NoError(t, err)require.False(t, ok)require.Empty(t, path)})}-func TestEnsureSQLiteFilesExistInternal(t *testing.T) {+func TestEnsureSQLiteFilesExist(t *testing.T) {tempDir := t.TempDir()dbPath := filepath.Join(tempDir, "arcane.db")- require.NoError(t, ensureSQLiteFilesExistInternal("file:"+dbPath))+ require.NoError(t, ensureSQLiteFilesExist("file:"+dbPath))require.FileExists(t, dbPath)require.NoFileExists(t, dbPath+"-wal")@@ -391,7 +391,7 @@ func TestEnsureSQLiteFilesExistInternalCreatesParentDir(t *testing.T) {tempDir := t.TempDir()dbPath := filepath.Join(tempDir, "subdir", "arcane.db")- require.NoError(t, ensureSQLiteFilesExistInternal("file:"+dbPath))+ require.NoError(t, ensureSQLiteFilesExist("file:"+dbPath))require.FileExists(t, dbPath)}@@ -411,7 +411,7 @@ func TestUnescapeMountInfoPath(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- require.Equal(t, tt.expect, unescapeMountInfoPathInternal(tt.input))+ require.Equal(t, tt.expect, unescapeMountInfoPath(tt.input))})}}@@ -446,7 +446,7 @@ func TestChownRecursiveInternalSkipsProjects(t *testing.T) {})mountpoints := make(map[string]struct{})- err := chownRecursiveInternal(tempDir, 1000, 1000, mountpoints, filepath.Clean(projectsDir))+ err := chownRecursive(tempDir, 1000, 1000, mountpoints, filepath.Clean(projectsDir))require.NoError(t, err)cleanedTempDir := filepath.Clean(tempDir)@@ -490,7 +490,7 @@ func TestPrepareWritablePathsInternalChownsTopLevelProjectsShallowly(t *testing.lchownFn = oldLchownFn})- err := prepareWritablePathsWithRootsInternal(os.Getuid(), os.Getgid(), map[string]struct{}{}, filepath.Clean(projectsDir), dataDir, buildsDir)+ err := prepareWritablePathsWithRoots(os.Getuid(), os.Getgid(), map[string]struct{}{}, filepath.Clean(projectsDir), dataDir, buildsDir)require.NoError(t, err)require.True(t, visited[filepath.Clean(projectsDir)], "top-level projects dir must be chowned")diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.goindex 0b007c936..3bdd04d89 100644--- a/backend/pkg/scheduler/runs/activity.go+++ b/backend/pkg/scheduler/runs/activity.go@@ -13,7 +13,7 @@ import (// SetObserver connects activity projection before accepting work.func (q *Coordinator) SetObserver(observer scheduler.RunObserver) { q.observer = observer }-func (q *Coordinator) associateActivityInternal(run *scheduler.Run) {+func (q *Coordinator) associateActivity(run *scheduler.Run) {if q.observer != nil && run.ActivityID == "" {run.ActivityID = q.observer.ActivityID(*run)}@@ -22,22 +22,22 @@ func (q *Coordinator) associateActivityInternal(run *scheduler.Run) {}}-func (q *Coordinator) observeRunInternal(ctx context.Context, run scheduler.Run) {+func (q *Coordinator) observeRun(ctx context.Context, run scheduler.Run) {if q.observer == nil {return}q.observerMu.Lock()defer q.observerMu.Unlock()- q.syncActivityInternal(ctx, run)- q.signalInternal()+ q.syncActivity(ctx, run)+ q.signal()}-func (q *Coordinator) syncActivityInternal(ctx context.Context, run scheduler.Run) {+func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) {ctx, cancel := context.WithTimeout(ctx, 5*time.Second)defer cancel()var state scheduler.CoordinatorStateid := kit.SHA256Hex(run.EnvironmentID + "\x00" + run.JobID)- if err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state); err != nil {+ if err := q.service.GetState(ctx, coordinatorType, id, &state); err != nil {return}version, pending := state.PendingActivitySync[run.ID]@@ -49,20 +49,20 @@ func (q *Coordinator) syncActivityInternal(ctx context.Context, run scheduler.Ruerr = q.observer.SyncRunActivity(ctx, current)}if err == nil {- _, err = q.service.Invoke(ctx, coordinatorTypeInternal, id, "activity-synced", scheduler.ActivitySyncCommand{RunID: run.ID, UpdatedAt: version})+ _, err = q.service.Invoke(ctx, coordinatorType, id, "activity-synced", scheduler.ActivitySyncCommand{RunID: run.ID, UpdatedAt: version})}if err != nil {slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)}}-func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {+func (q *Coordinator) retryActivitySync(ctx context.Context) {if q.observer == nil {return}cursor := ""for {- page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})+ page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})if err != nil {slog.ErrorContext(ctx, "List pending job activities", "error", err)return@@ -78,7 +78,7 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {return}q.observerMu.Lock()- q.syncActivityInternal(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})+ q.syncActivity(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})q.observerMu.Unlock()}}@@ -89,12 +89,12 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {}}-func (q *Coordinator) hasPendingActivitySyncInternal(ctx context.Context) bool {+func (q *Coordinator) hasPendingActivitySync(ctx context.Context) bool {ctx, cancel := context.WithTimeout(ctx, 5*time.Second)defer cancel()cursor := ""for {- page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})+ page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})if err != nil {return true}diff --git a/backend/pkg/scheduler/runs/activity_test.go b/backend/pkg/scheduler/runs/activity_test.goindex aad9002c9..8113b4c1c 100644--- a/backend/pkg/scheduler/runs/activity_test.go+++ b/backend/pkg/scheduler/runs/activity_test.go@@ -11,20 +11,20 @@ import ("github.com/stretchr/testify/require")-type activityObserverFakeInternal struct {+type activityObserverFake struct {unavailable atomic.Boolobserved atomic.Int32existingOnly bool}-func (o *activityObserverFakeInternal) ActivityID(run st.Run) string {+func (o *activityObserverFake) ActivityID(run st.Run) string {if o.existingOnly {return run.ActivityID}return run.ID}-func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run) error {+func (o *activityObserverFake) SyncRunActivity(context.Context, st.Run) error {if o.unavailable.Load() {return errors.New("activity storage unavailable")}@@ -33,8 +33,8 @@ func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run)}func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {- q := newResolutionQueueInternal(t)- observer := &activityObserverFakeInternal{}+ q := newResolutionQueue(t)+ observer := &activityObserverFake{}observer.unavailable.Store(true)q.SetObserver(observer)var executions atomic.Int32@@ -56,10 +56,10 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {persisted, getErr := q.Get(t.Context(), "0", run.JobID, run.ID)return getErr == nil && persisted.Status == st.Succeeded}, 5*time.Second, 10*time.Millisecond)- require.True(t, q.hasPendingActivitySyncInternal(t.Context()))+ require.True(t, q.hasPendingActivitySync(t.Context()))observer.unavailable.Store(false)- q.retryActivitySyncInternal(t.Context())- require.False(t, q.hasPendingActivitySyncInternal(t.Context()))+ q.retryActivitySync(t.Context())+ require.False(t, q.hasPendingActivitySync(t.Context()))require.Positive(t, observer.observed.Load())duplicate, err := q.Submit(t.Context(), st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "manual"})require.NoError(t, err)diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.goindex 1f6d33a41..1201c95a7 100644--- a/backend/pkg/scheduler/runs/actors.go+++ b/backend/pkg/scheduler/runs/actors.go@@ -1,6 +1,7 @@package runsimport (+ "cmp""context""encoding/json/v2""errors"@@ -22,51 +23,51 @@ import ()const (- coordinatorTypeInternal = "job-coordinator"- executorTypeInternal = "job-executor"- healthExecutorTypeInternal = "job-health-executor"- orphanedDispatchAge = 15 * time.Minute- outcomePersistGrace = 30 * time.Second+ coordinatorType = "job-coordinator"+ executorType = "job-executor"+ healthExecutorType = "job-health-executor"+ orphanedDispatchAge = 15 * time.Minute+ outcomePersistGrace = 30 * time.Second)-type coordinatorActorInternal struct {+type coordinatorActor struct {id stringq *Coordinator}-type executorActorInternal struct {+type executorActor struct {q *Coordinator}func (q *Coordinator) Register(runtime *francis.Runtime) error {q.runtime = runtime- if err := runtime.RegisterActor(coordinatorTypeInternal, func(id string, _ *actor.Service) actor.Actor {- return &coordinatorActorInternal{id: id, q: q}+ if err := runtime.RegisterActor(coordinatorType, func(id string, _ *actor.Service) actor.Actor {+ return &coordinatorActor{id: id, q: q}}); err != nil {return err}executorFactory := func(_ string, _ *actor.Service) actor.Actor {- return &executorActorInternal{q: q}+ return &executorActor{q: q}}if err := runtime.RegisterActor(- executorTypeInternal, executorFactory,+ executorType, executorFactory,local.WithCapacityGroup("jobs", 4),local.WithCompletedJobRetention(7*24*time.Hour),); err != nil {return err}return runtime.RegisterActor(- healthExecutorTypeInternal, executorFactory,+ healthExecutorType, executorFactory,local.WithCapacityGroup("jobs-health", 2),local.WithCompletedJobRetention(7*24*time.Hour),)}-func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {+func (a *coordinatorActor) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {var state st.CoordinatorState- if err := a.q.service.GetState(ctx, coordinatorTypeInternal, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ if err := a.q.service.GetState(ctx, coordinatorType, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {return nil, err}- normalizeRecordTimesInternal(&state.Record)+ normalizeRecordTimes(&state.Record)switch method {case "replace":var next st.CoordinatorState@@ -77,13 +78,13 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, dareturn false, nil}next.Revision++- if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, next, nil); err != nil {+ if err := a.q.service.SetState(ctx, coordinatorType, a.id, next, nil); err != nil {return nil, err}// Acceptance is persisted; dispatch failures are retried.- if err := a.flushInternal(ctx, &next); err != nil {+ if err := a.flush(ctx, &next); err != nil {slog.WarnContext(ctx, "Job dispatch requires reconciliation", "jobId", next.Record.JobID, "environmentId", next.Record.EnvironmentID, "error", err)- a.q.signalInternal()+ a.q.signal()}return true, nilcase "activity-synced":@@ -97,9 +98,9 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, da}delete(state.PendingActivitySync, command.RunID)state.Revision++- return nil, a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil)+ return nil, a.q.service.SetState(ctx, coordinatorType, a.id, state, nil)case "repair":- return nil, a.flushInternal(ctx, &state)+ return nil, a.flush(ctx, &state)case "import":var command st.CoordinatorImportif err := data.Decode(&command); err != nil {@@ -117,17 +118,17 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, dastate.ImportHash = command.SourceHashstate.Record = command.Recordfor _, run := range state.Record.Runs {- markActivitySyncInternal(&state, run)+ markActivitySync(&state, run)}state.Imported = truestate.Revision = 1- return true, a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil)+ return true, a.q.service.SetState(ctx, coordinatorType, a.id, state, nil)default:return nil, fmt.Errorf("unknown coordinator command: %s", method)}}-func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.CoordinatorState) error {+func (a *coordinatorActor) flush(ctx context.Context, state *st.CoordinatorState) error {if !a.q.enabled.Load() {return nil}@@ -138,7 +139,7 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.DueTime: record.NextRun,Data: st.ScheduleOccurrence{Generation: record.Generation, Sequence: record.Sequence},}- if err := a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, properties); err != nil {+ if err := a.q.service.SetAlarm(ctx, coordinatorType, a.id, name, properties); err != nil {return err}}@@ -151,14 +152,14 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.}var retryAt time.Timeif selected == nil {- selected, retryAt = a.q.nextRunInternal(record.Runs)+ selected, retryAt = a.q.nextRun(record.Runs)}if selected == nil {if retryAt.IsZero() {return nil}name := fmt.Sprintf("retry-%d", retryAt.UnixNano())- return a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, actor.AlarmProperties{DueTime: retryAt})+ return a.q.service.SetAlarm(ctx, coordinatorType, a.id, name, actor.AlarmProperties{DueTime: retryAt})}if selected.Status == st.Running {active, err := a.reconcileDispatch(ctx, state, *selected)@@ -173,7 +174,7 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.}// reconcileDispatch recovers a lost job ID before considering redelivery.-func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {+func (a *coordinatorActor) reconcileDispatch(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {key := fmt.Sprintf("%s-%d", run.ID, run.AttemptCount)previous, exists := state.Dispatches[key]if !exists {@@ -181,8 +182,8 @@ func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state}actorType := kit.Ternary(strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",- healthExecutorTypeInternal,- executorTypeInternal,+ healthExecutorType,+ executorType,)if previous.JobID == "" {command := st.ExecutionCommand{@@ -198,7 +199,7 @@ func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, stateprevious.JobID = idstate.Dispatches[key] = previousstate.Revision++- if setStateErr := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil); setStateErr != nil {+ if setStateErr := a.q.service.SetState(ctx, coordinatorType, a.id, *state, nil); setStateErr != nil {return false, setStateErr}}@@ -219,15 +220,15 @@ func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, statereturn false, nil}-func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {+func (a *coordinatorActor) dispatchRun(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {key := fmt.Sprintf("%s-%d", run.ID, attempt)if state.Dispatches == nil {state.Dispatches = make(map[string]st.DispatchIntent)}actorType := kit.Ternary(strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",- healthExecutorTypeInternal,- executorTypeInternal,+ healthExecutorType,+ executorType,)intent := state.Dispatches[key]if intent.JobID != "" {@@ -244,10 +245,9 @@ func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.Co}}}- intent = st.DispatchIntent{RunID: run.ID, Attempt: attempt}- state.Dispatches[key] = intent+ state.Dispatches[key] = st.DispatchIntent{RunID: run.ID, Attempt: attempt}state.Revision++- if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil); err != nil {+ if err := a.q.service.SetState(ctx, coordinatorType, a.id, *state, nil); err != nil {return err}options := []actor.JobOption{actor.WithIdempotencyKey(key)}@@ -277,23 +277,22 @@ func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.Coreturn err}}- intent.JobID = id- state.Dispatches[key] = intent+ state.Dispatches[key] = st.DispatchIntent{RunID: run.ID, Attempt: attempt, JobID: id}state.Revision++- return a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil)+ return a.q.service.SetState(ctx, coordinatorType, a.id, *state, nil)}-func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data actor.Envelope) error {+func (a *coordinatorActor) Alarm(ctx context.Context, name string, data actor.Envelope) error {if !a.q.enabled.Load() {return nil}var state st.CoordinatorState- if err := a.q.service.GetState(ctx, coordinatorTypeInternal, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {+ if err := a.q.service.GetState(ctx, coordinatorType, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {return err}- normalizeRecordTimesInternal(&state.Record)+ normalizeRecordTimes(&state.Record)if strings.HasPrefix(name, "retry-") {- return a.flushInternal(ctx, &state)+ return a.flush(ctx, &state)}var occurrence st.ScheduleOccurrenceif err := data.Decode(&occurrence); err != nil {@@ -319,17 +318,9 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, dataif !nextOccurrence.After(now) {trigger = "recovery"}- pending := false- for _, run := range record.Runs {- if separateTriggerInternal(run.Trigger) {- continue- }- if run.Status.Terminal() || run.Status == st.NeedsAttention {- continue- }- pending = true- break- }+ pending := slices.ContainsFunc(record.Runs, func(run st.Run) bool {+ return !separateTrigger(run.Trigger) && !run.Status.Terminal() && run.Status != st.NeedsAttention+ })if !pending {run := st.Run{ID: uuid.New().String(),@@ -340,23 +331,23 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, dataCreatedAt: now,UpdatedAt: now,}- a.q.associateActivityInternal(&run)+ a.q.associateActivity(&run)record.Runs = append(record.Runs, run)- markActivitySyncInternal(&state, run)+ markActivitySync(&state, run)}record.LastEnqueuedAt = nowrecord.NextRun = schedule.Next(now.In(a.q.location))record.Sequence++state.Revision++- if setStateErr := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil); setStateErr != nil {+ if setStateErr := a.q.service.SetState(ctx, coordinatorType, a.id, state, nil); setStateErr != nil {return setStateErr}- return a.flushInternal(ctx, &state)+ return a.flush(ctx, &state)}-// repairInternal flushes coordinators so schedule alarms and dispatches are restored.+// repair flushes coordinators so schedule alarms and dispatches are restored.// Unless all is set, it skips records without unfinished runs so idle coordinators can deactivate.-func (q *Coordinator) repairInternal(ctx context.Context, all bool) error {+func (q *Coordinator) repair(ctx context.Context, all bool) error {records, err := q.Records(ctx)if err != nil {return err@@ -365,15 +356,15 @@ func (q *Coordinator) repairInternal(ctx context.Context, all bool) error {if !all && !slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention }) {continue}- if _, invokeErr := q.service.Invoke(ctx, coordinatorTypeInternal, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {+ if _, invokeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {return invokeErr}}return nil}-func (q *Coordinator) importLegacyInternal(ctx context.Context) error {- entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)+func (q *Coordinator) importLegacy(ctx context.Context) error {+ entries, err := q.store.ListByPrefix(ctx, queuePrefix)if err != nil {return fmt.Errorf("list legacy job records: %w", err)}@@ -391,16 +382,16 @@ func (q *Coordinator) importLegacyInternal(ctx context.Context) error {return fmt.Errorf("decode legacy job record %q: %w", entry.Key, unmarshalErr)}id := kit.SHA256Hex(record.EnvironmentID + "\x00" + record.JobID)- if entry.Key != queuePrefixInternal+id {+ if entry.Key != queuePrefix+id {return fmt.Errorf("invalid legacy job key %q for job %q in environment %q", entry.Key, record.JobID, record.EnvironmentID)}sourceHash := kit.SHA256Hex(entry.Value)command := st.CoordinatorImport{Record: record, SourceHash: sourceHash}- if _, invokeErr := q.service.Invoke(ctx, coordinatorTypeInternal, id, "import", command); invokeErr != nil {+ if _, invokeErr := q.service.Invoke(ctx, coordinatorType, id, "import", command); invokeErr != nil {return fmt.Errorf("import legacy job record %q: %w", entry.Key, invokeErr)}var imported st.CoordinatorState- if getStateErr := q.service.GetState(ctx, coordinatorTypeInternal, id, &imported); getStateErr != nil {+ if getStateErr := q.service.GetState(ctx, coordinatorType, id, &imported); getStateErr != nil {return fmt.Errorf("import legacy job record %q: %w", entry.Key, getStateErr)}if !imported.Imported || imported.ImportHash != sourceHash {@@ -416,7 +407,7 @@ func (q *Coordinator) importLegacyInternal(ctx context.Context) error {return nil}-func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Envelope) error {+func (a *executorActor) Job(ctx context.Context, _ string, data actor.Envelope) error {q := a.qq.mu.Lock()lifetime := q.lifecycleCtx@@ -438,7 +429,7 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Enif err := data.Decode(&command); err != nil {return errors.Join(err, actor.ErrJobPermanentFailure)}- claim, err := q.claimExecutionInternal(ctx, command)+ claim, err := q.claimExecution(ctx, command)if err != nil {return err}@@ -448,38 +439,43 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Enif !claim.Claimed {return nil}- outcome, runErr := q.invokeInternal(ctx, claim.Run, claim.Reconcile)- if outcome.Status == "" {- if runErr != nil {- outcome.Status = st.Failed- } else {- outcome.Status = st.Succeeded+ outcome, runErr := func() (outcome st.Outcome, err error) {+ defer func() {+ if recovered := recover(); recovered != nil {+ outcome = st.Outcome{Status: st.Failed, Message: "Job panicked; inspect logs before retrying"}+ err = fmt.Errorf("job panic: %v", recovered)+ slog.ErrorContext(ctx, "Job panicked", "runId", claim.Run.ID, "error", err)+ }+ }()+ if claim.Reconcile && q.reconcile != nil {+ return q.reconcile(ctx, claim.Run)}- }+ return q.execute(ctx, claim.Run)+ }()// Domain recovery may still report uncertainty; job runs finish as failures.- if outcome.Status == st.NeedsAttention {- outcome.Status = st.Failed- }- if runErr != nil && outcome.Message == "" {- outcome.Message = runErr.Error()+ status := cmp.Or(outcome.Status, kit.Ternary(runErr != nil, st.Failed, st.Succeeded))+ outcome = st.Outcome{+ Status: kit.Ternary(status == st.NeedsAttention, st.Failed, status),+ Message: cmp.Or(outcome.Message, kit.Ternary(runErr != nil, fmt.Sprint(runErr), "")),+ ActivityID: outcome.ActivityID,+ Targets: outcome.Targets,+ Steps: outcome.Steps,}if runErr != nil {if ctx.Err() != nil {return ctx.Err()}- q.persistOutcome(ctx, claim.Run, outcome)- return ctx.Err()+ return cmp.Or(q.persistOutcome(ctx, claim.Run, outcome), ctx.Err())}persistCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))defer cancel()defer context.AfterFunc(ctx, func() { time.AfterFunc(outcomePersistGrace, cancel) })()- q.persistOutcome(persistCtx, claim.Run, outcome)- return ctx.Err()+ return cmp.Or(q.persistOutcome(persistCtx, claim.Run, outcome), ctx.Err())}-func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {+func (q *Coordinator) claimExecution(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {var claim st.ExecutionClaim- err := q.mutateInternal(ctx, command.EnvironmentID, command.JobID, func(record *st.QueueRecord) error {+ err := q.mutate(ctx, command.EnvironmentID, command.JobID, func(record *st.QueueRecord) error {claim = st.ExecutionClaim{}for i := range record.Runs {current := &record.Runs[i]@@ -498,10 +494,10 @@ func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.Exe// Executor turns serialize attempts for this coordinator.now := time.Now().UTC()if current.NextAttempt != nil && current.NextAttempt.After(now) {- claim.Deferred = true+ claim = st.ExecutionClaim{Deferred: true}return nil}- claim.Reconcile = current.Status == st.Running+ reconcile := current.Status == st.Runningcurrent.Status = st.Runningcurrent.Owner = q.owner + "/" + uuid.New().String()current.UpdatedAt = now@@ -515,8 +511,11 @@ func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.Exeif len(current.Attempts) > 100 {current.Attempts = current.Attempts[len(current.Attempts)-100:]}- claim.Run = *current- claim.Claimed = true+ claim = st.ExecutionClaim{+ Run: *current,+ Claimed: true,+ Reconcile: reconcile,+ }return nil}return nildiff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex d3f406091..44513a751 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -69,20 +69,20 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {}data, marshalErr := json.Marshal(record)require.NoError(t, marshalErr)- require.NoError(t, store.Set(t.Context(), queuePrefixInternal+kit.SHA256Hex("0\x00"+jobID), string(data)))+ require.NoError(t, store.Set(t.Context(), queuePrefix+kit.SHA256Hex("0\x00"+jobID), string(data)))}encoded, err := json.Marshal(legacy)require.NoError(t, err)- key := queuePrefixInternal + kit.SHA256Hex("0\x00"+legacy.JobID)+ key := queuePrefix + kit.SHA256Hex("0\x00"+legacy.JobID)require.NoError(t, store.Set(t.Context(), key, string(encoded)))databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))first := francistest.New(t, databaseURL)q := New(store, first.Service(), time.UTC)require.NoError(t, q.Register(first))francistest.Start(t, first)- require.NoError(t, q.importLegacyInternal(t.Context()))- require.NoError(t, q.importLegacyInternal(t.Context()))- observer := &activityObserverFakeInternal{existingOnly: true}+ require.NoError(t, q.importLegacy(t.Context()))+ require.NoError(t, q.importLegacy(t.Context()))+ observer := &activityObserverFake{existingOnly: true}observer.unavailable.Store(true)q.SetObserver(observer)_, err = q.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})@@ -91,7 +91,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {run.Outcome.Targets = []st.TargetOutcome{{ID: "artifact", Status: st.Succeeded, RecoveryData: jsontext.Value(`{"backupId":"artifact"}`)}}return nil}))- require.True(t, q.hasPendingActivitySyncInternal(t.Context()))+ require.True(t, q.hasPendingActivitySync(t.Context()))q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {t.Error("startup must not execute jobs before activation")return st.Outcome{Status: st.Failed}, nil@@ -106,7 +106,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {restored.SetExecutor(q.execute, nil)require.NoError(t, restored.Register(second))francistest.Start(t, second)- require.NoError(t, restored.importLegacyInternal(t.Context()))+ require.NoError(t, restored.importLegacy(t.Context()))require.NoError(t, restored.Start(t.Context(), t.Context()))t.Cleanup(func() { require.NoError(t, restored.Stop(context.WithoutCancel(t.Context()))) })for _, jobID := range []string{"gitops-sync:first", "gitops-sync:second"} {@@ -128,31 +128,31 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {require.Equal(t, 1, history.Total)require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())observer.unavailable.Store(false)- restored.retryActivitySyncInternal(t.Context())- require.False(t, restored.hasPendingActivitySyncInternal(t.Context()))+ restored.retryActivitySync(t.Context())+ require.False(t, restored.hasPendingActivitySync(t.Context()))require.Positive(t, observer.observed.Load())_, exists, err := store.Get(t.Context(), key)require.NoError(t, err)require.True(t, exists)}-type occurrenceEnvelopeInternal struct{ occurrence st.ScheduleOccurrence }+type occurrenceEnvelope struct{ occurrence st.ScheduleOccurrence }-func (e occurrenceEnvelopeInternal) Decode(value any) error {+func (e occurrenceEnvelope) Decode(value any) error {*value.(*st.ScheduleOccurrence) = e.occurrencereturn nil}func TestAlarmIgnoresStaleGeneration(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)next := time.Now().UTC().Add(time.Hour)require.NoError(t, q.Checkpoint(t.Context(), "schedule", "0 * * * * *", next))before, err := q.ScheduleState(t.Context(), "schedule")require.NoError(t, err)q.enabled.Store(true)defer q.enabled.Store(false)- coordinator := &coordinatorActorInternal{id: kit.SHA256Hex("0\x00schedule"), q: q}- require.NoError(t, coordinator.Alarm(t.Context(), "old", occurrenceEnvelopeInternal{st.ScheduleOccurrence{Generation: before.Generation - 1, Sequence: before.Sequence}}))+ coordinator := &coordinatorActor{id: kit.SHA256Hex("0\x00schedule"), q: q}+ require.NoError(t, coordinator.Alarm(t.Context(), "old", occurrenceEnvelope{st.ScheduleOccurrence{Generation: before.Generation - 1, Sequence: before.Sequence}}))after, err := q.ScheduleState(t.Context(), "schedule")require.NoError(t, err)require.Equal(t, before.Generation, after.Generation)@@ -168,7 +168,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {name = "canceled"}t.Run(name, func(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)var executed atomic.Int32q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {executed.Add(1)@@ -178,10 +178,10 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {require.NoError(t, err)id := kit.SHA256Hex("0\x00" + run.JobID)command := st.ExecutionCommand{EnvironmentID: "0", JobID: run.JobID, RunID: run.ID, Attempt: 1}- jobID, created, err := q.service.Dispatch(t.Context(), executorTypeInternal, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))+ jobID, created, err := q.service.Dispatch(t.Context(), executorType, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))require.NoError(t, err)require.True(t, created)- duplicateID, created, err := q.service.Dispatch(t.Context(), executorTypeInternal, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))+ duplicateID, created, err := q.service.Dispatch(t.Context(), executorType, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))require.NoError(t, err)require.False(t, created)require.Equal(t, jobID, duplicateID)@@ -215,7 +215,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {}func TestCapacityGroupsReserveHealthSlots(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)var ordinary, health atomic.Int32release := make(chan struct{})var once sync.Once@@ -261,19 +261,19 @@ func TestCapacityGroupsReserveHealthSlots(t *testing.T) {}func TestReconciliationRepairsMissingAlarm(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)require.NoError(t, q.Checkpoint(t.Context(), "repair", "0 * * * * *", time.Now().UTC().Add(time.Hour)))q.enabled.Store(true)defer q.enabled.Store(false)record, err := q.ScheduleState(t.Context(), "repair")require.NoError(t, err)id := kit.SHA256Hex("0\x00repair")- err = q.repairInternal(t.Context(), true)+ err = q.repair(t.Context(), true)require.NoError(t, err)- require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))- err = q.repairInternal(t.Context(), true)+ require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))+ err = q.repair(t.Context(), true)require.NoError(t, err)- require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))+ require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))after, err := q.ScheduleState(t.Context(), "repair")require.NoError(t, err)require.Equal(t, record.Generation, after.Generation)@@ -307,12 +307,12 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {require.True(t, admitted)replacement.Release(t.Context())- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)run, err := q.Submit(t.Context(), st.Request{JobID: "tokens"})require.NoError(t, err)require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error { current.Status = st.Running; current.Owner = "new-token"; return nil }))run.Owner = "stale-token"- q.persistOutcome(t.Context(), run, st.Outcome{Status: st.Succeeded})+ require.NoError(t, q.persistOutcome(t.Context(), run, st.Outcome{Status: st.Succeeded}))persisted, err := q.Get(t.Context(), "0", run.JobID, run.ID)require.NoError(t, err)require.Equal(t, st.Running, persisted.Status)@@ -320,7 +320,7 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {}func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)entered := make(chan struct{})finish := make(chan struct{})var once sync.Once@@ -349,7 +349,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {}id := kit.SHA256Hex("0\x00" + run.JobID)var state st.CoordinatorState- require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))+ require.NoError(t, q.service.GetState(t.Context(), coordinatorType, id, &state))key := run.ID + "-1"original := state.Dispatches[key].JobIDrequire.NotEmpty(t, original)@@ -357,12 +357,12 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {intent.JobID = ""state.Dispatches[key] = intentstate.Revision++- require.NoError(t, q.service.SetState(t.Context(), coordinatorTypeInternal, id, state, nil))- err = q.repairInternal(t.Context(), false)+ require.NoError(t, q.service.SetState(t.Context(), coordinatorType, id, state, nil))+ err = q.repair(t.Context(), false)require.NoError(t, err)- require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))+ require.NoError(t, q.service.GetState(t.Context(), coordinatorType, id, &state))require.Equal(t, original, state.Dispatches[key].JobID)- deliveries, err := q.service.ListJobs(t.Context(), executorTypeInternal, id)+ deliveries, err := q.service.ListJobs(t.Context(), executorType, id)require.NoError(t, err)require.Len(t, deliveries, 1)once.Do(func() { close(finish) })diff --git a/backend/pkg/scheduler/runs/admission.go b/backend/pkg/scheduler/runs/admission.goindex d8da20767..73a2d9158 100644--- a/backend/pkg/scheduler/runs/admission.go+++ b/backend/pkg/scheduler/runs/admission.go@@ -15,7 +15,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis")-const admissionTypeInternal = "resource-admission"+const admissionType = "resource-admission"type Admission struct {service *actor.Service@@ -29,7 +29,7 @@ type Lease struct {token stringonce sync.Once}-type admissionActorInternal struct {+type admissionActor struct {id stringservice *actor.Service}@@ -39,8 +39,8 @@ func NewAdmission(service *actor.Service, epoch string) *Admission {}func (a *Admission) Register(runtime *francis.Runtime) error {- return runtime.RegisterActor(admissionTypeInternal, func(id string, service *actor.Service) actor.Actor {- return &admissionActorInternal{id: id, service: service}+ return runtime.RegisterActor(admissionType, func(id string, service *actor.Service) actor.Actor {+ return &admissionActor{id: id, service: service}})}@@ -50,20 +50,20 @@ func (a *Admission) TryAcquire(ctx context.Context, key st.AdmissionKey) (*Leaseif err := ctx.Err(); err != nil {return nil, false, err}- if err := a.retryReleasesInternal(ctx, id); err != nil {+ if err := a.retryReleases(ctx, id); err != nil {return nil, false, err}// Finish admission after submission so cancellation cannot orphan a lease.invokeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)defer cancel()- envelope, err := a.service.Invoke(invokeCtx, admissionTypeInternal, id, "acquire", st.AdmissionCommand{Epoch: a.epoch, Token: token})+ envelope, err := a.service.Invoke(invokeCtx, admissionType, id, "acquire", st.AdmissionCommand{Epoch: a.epoch, Token: token})if err != nil {- a.releaseInternal(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})+ a.release(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})return nil, false, err}var acquired boolif decodeErr := envelope.Decode(&acquired); decodeErr != nil {- a.releaseInternal(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})+ a.release(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})return nil, false, decodeErr}if !acquired {@@ -77,11 +77,11 @@ func (l *Lease) Release(ctx context.Context) {return}l.once.Do(func() {- l.admission.releaseInternal(ctx, l.id, st.AdmissionCommand{Epoch: l.admission.epoch, Token: l.token})+ l.admission.release(ctx, l.id, st.AdmissionCommand{Epoch: l.admission.epoch, Token: l.token})})}-func (a *Admission) releaseInternal(ctx context.Context, id string, command st.AdmissionCommand) {+func (a *Admission) release(ctx context.Context, id string, command st.AdmissionCommand) {a.mu.Lock()if a.pending[id] == nil {a.pending[id] = make(map[string]st.AdmissionCommand)@@ -90,12 +90,12 @@ func (a *Admission) releaseInternal(ctx context.Context, id string, command st.Aa.mu.Unlock()ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)defer cancel()- if err := a.retryReleasesInternal(ctx, id); err != nil {+ if err := a.retryReleases(ctx, id); err != nil {slog.ErrorContext(ctx, "Resource admission release failed; will retry before further admission", "resource", id, "error", err)}}-func (a *Admission) retryReleasesInternal(ctx context.Context, id string) error {+func (a *Admission) retryReleases(ctx context.Context, id string) error {a.mu.Lock()pending := make([]st.AdmissionCommand, 0, len(a.pending[id]))for _, command := range a.pending[id] {@@ -103,7 +103,7 @@ func (a *Admission) retryReleasesInternal(ctx context.Context, id string) error}a.mu.Unlock()for _, command := range pending {- if _, err := a.service.Invoke(ctx, admissionTypeInternal, id, "release", command); err != nil {+ if _, err := a.service.Invoke(ctx, admissionType, id, "release", command); err != nil {return err}a.mu.Lock()@@ -116,9 +116,9 @@ func (a *Admission) retryReleasesInternal(ctx context.Context, id string) errorreturn nil}-func (a *admissionActorInternal) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {+func (a *admissionActor) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {var state st.AdmissionState- err := a.service.GetState(ctx, admissionTypeInternal, a.id, &state)+ err := a.service.GetState(ctx, admissionType, a.id, &state)if err != nil && !errors.Is(err, actor.ErrStateNotFound) {return nil, err}@@ -140,7 +140,7 @@ func (a *admissionActorInternal) Invoke(ctx context.Context, method string, datadefault:return nil, errors.New("unknown resource admission command")}- if setStateErr := a.service.SetState(ctx, admissionTypeInternal, a.id, state, nil); setStateErr != nil {+ if setStateErr := a.service.SetState(ctx, admissionType, a.id, state, nil); setStateErr != nil {return nil, setStateErr}return true, nildiff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.goindex 1242a90b7..a97c668f4 100644--- a/backend/pkg/scheduler/runs/coordinator.go+++ b/backend/pkg/scheduler/runs/coordinator.go@@ -63,9 +63,7 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, ereturn st.Run{}, errors.New("job ID required")}request.Trigger = cmp.Or(request.Trigger, "manual")- if request.RunID == "" {- request.RunID = uuid.New().String()- }+ request.RunID = cmp.Or(request.RunID, uuid.New().String())if _, err := uuid.Parse(request.RunID); err != nil {return st.Run{}, errors.New("invalid run ID")}@@ -84,8 +82,13 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, ereturn st.Run{}, errors.New("observed agent run is not awaiting legacy resolution")}observed = *agentRun- remoteOutcome = new(observed.Outcome)- remoteOutcome.Status = observed.Status+ remoteOutcome = &st.Outcome{+ Status: observed.Status,+ Message: observed.Outcome.Message,+ ActivityID: observed.Outcome.ActivityID,+ Targets: observed.Outcome.Targets,+ Steps: observed.Outcome.Steps,+ }}result := st.Run{ID: request.RunID,@@ -106,13 +109,16 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, eRemoteAttemptCount: observed.AttemptCount,RemoteOutcome: remoteOutcome,}- q.associateActivityInternal(&result)- err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {+ q.associateActivity(&result)+ err := q.mutate(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {if request.Trigger == "scheduled" || request.Trigger == "recovery" {record.LastEnqueuedAt = now}for _, existing := range record.Runs {- if existing.ID == request.RunID || coalescesPendingInternal(request, existing) {+ // Manual and remote work never joins another run; other triggers join runs still waiting to start.+ coalesces := request.ObservedAgentRun == nil && !separateTrigger(request.Trigger) && !separateTrigger(existing.Trigger) &&+ (existing.Status == st.Queued || existing.Status == st.Waiting || existing.Status == st.Retrying)+ if existing.ID == request.RunID || coalesces {result = existingreturn nil}@@ -121,34 +127,25 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, eresult = existingreturn nil}- pruneRunsInternal(record, now)+ pruneRuns(record, now)record.Runs = append(record.Runs, result)return nil})if err == nil {- q.observeRunInternal(ctx, result)- q.signalInternal()+ q.observeRun(ctx, result)+ q.signal()}return result, err}-func (q *Coordinator) signalInternal() {+func (q *Coordinator) signal() {select {case q.wake <- struct{}{}:default:}}-// Manual and remote work never joins another run. Scheduled and recovery work-// joins an existing run only while it is still waiting to start.-func coalescesPendingInternal(request st.Request, existing st.Run) bool {- if request.ObservedAgentRun != nil || separateTriggerInternal(request.Trigger) || separateTriggerInternal(existing.Trigger) {- return false- }- return existing.Status == st.Queued || existing.Status == st.Waiting || existing.Status == st.Retrying-}--func separateTriggerInternal(trigger string) bool {+func separateTrigger(trigger string) bool {switch trigger {case "manual", "remote":return true@@ -164,22 +161,23 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, nefirst := !q.checkpointed[jobID]recovered := falsevar recoveredRun st.Run- err := q.mutateInternal(ctx, "0", jobID, func(record *st.QueueRecord) error {+ err := q.mutate(ctx, "0", jobID, func(record *st.QueueRecord) error {recovered = falserecoveredRun = st.Run{}now := time.Now().UTC()if first && record.Schedule == schedule && !record.NextRun.IsZero() && !record.NextRun.After(now) && record.LastEnqueuedAt.Before(record.NextRun) {- pending := false- for _, run := range record.Runs {- if !run.Status.Terminal() && run.Status != st.NeedsAttention {- pending = true- break- }- }- if !pending {+ if !slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention }) {recovered = truerecord.LastEnqueuedAt = now- recoveredRun = st.Run{ID: uuid.New().String(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now}+ recoveredRun = st.Run{+ ID: uuid.New().String(),+ JobID: jobID,+ EnvironmentID: "0",+ Trigger: "recovery",+ Status: st.Queued,+ CreatedAt: now,+ UpdatedAt: now,+ }record.Runs = append(record.Runs, recoveredRun)}}@@ -194,8 +192,8 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, neif err == nil {q.checkpointed[jobID] = trueif recovered {- q.observeRunInternal(ctx, recoveredRun)- q.signalInternal()+ q.observeRun(ctx, recoveredRun)+ q.signal()}}return err@@ -218,7 +216,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {if q.execute == nil {return errors.New("job executor unavailable")}- if err := q.importLegacyInternal(ctx); err != nil {+ if err := q.importLegacy(ctx); err != nil {return fmt.Errorf("import legacy job history: %w", err)}// Fail startup if persisted state cannot be read. Never replace it with empty state.@@ -229,7 +227,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {for _, record := range records {changed := falsefor _, run := range record.Runs {- if normalizeLegacyRunInternal(&run) {+ if normalizeLegacyRun(&run) {changed = truebreak}@@ -237,16 +235,16 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {if !changed {continue}- if mutateErr := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {+ if mutateErr := q.mutate(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {for i := range current.Runs {- normalizeLegacyRunInternal(¤t.Runs[i])+ normalizeLegacyRun(¤t.Runs[i])}return nil}); mutateErr != nil {return fmt.Errorf("normalize startup job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)}}- if pruneErr := q.pruneInternal(ctx); pruneErr != nil {+ if pruneErr := q.prune(ctx); pruneErr != nil {return fmt.Errorf("prune startup job history: %w", pruneErr)}if errErr := ctx.Err(); errErr != nil {@@ -260,7 +258,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {q.cancel = cancelq.lifecycleCtx = runCtxq.started = true- go q.dispatchInternal(runCtx)+ go q.dispatch(runCtx)return nil}@@ -286,7 +284,7 @@ func (q *Coordinator) Stop(ctx context.Context) error {}}-func (q *Coordinator) dispatchInternal(ctx context.Context) {+func (q *Coordinator) dispatch(ctx context.Context) {defer close(q.done)initialRetention := true@@ -294,20 +292,20 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {fullRepair := truenextRetention := time.Now()for ctx.Err() == nil {- q.retryActivitySyncInternal(ctx)+ q.retryActivitySync(ctx)if initialRetention || !time.Now().Before(nextRetention) {- if err := q.pruneInternal(ctx); err != nil {+ if err := q.prune(ctx); err != nil {if ctx.Err() == nil {slog.ErrorContext(ctx, "Job retention failed", "error", err)}} else {- initialRetention = q.hasPendingActivitySyncInternal(ctx)+ initialRetention = q.hasPendingActivitySync(ctx)}nextRetention = time.Now().Add(time.Hour)}nextWake := time.Now().Add(15 * time.Second)enabled := q.enabled.Load()- if err := q.repairInternal(ctx, fullRepair); err != nil {+ if err := q.repair(ctx, fullRepair); err != nil {if ctx.Err() == nil {slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)}@@ -318,7 +316,7 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {if nextRetention.Before(nextWake) {nextWake = nextRetention}- if q.hasPendingActivitySyncInternal(ctx) {+ if q.hasPendingActivitySync(ctx) {activityRetry := time.Now().Add(5 * time.Second)if activityRetry.Before(nextWake) {nextWake = activityRetry@@ -336,9 +334,10 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {}}-func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) {- // Keep the execution slot until the terminal result has been durably recorded.- for {+func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) error {+ // Keep the execution slot while retrying, but release it after a bounded backoff.+ wait := time.Second+ for attempt := 1; ; attempt++ {var conflictOwner stringvar conflictStatus st.RunStatuserr := q.UpdateRun(ctx, run, func(current *st.Run) error {@@ -349,7 +348,15 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome stnow := time.Now().UTC()current.Status = outcome.Statuscurrent.UpdatedAt = now- outcome = mergeTargetProgressInternal(current.Outcome, outcome)+ for _, target := range current.Outcome.Targets {+ if !slices.ContainsFunc(outcome.Targets, func(updated st.TargetOutcome) bool { return updated.ID == target.ID }) {+ outcome.Targets = append(outcome.Targets, target)+ }+ }+ outcome.ActivityID = cmp.Or(outcome.ActivityID, current.Outcome.ActivityID)+ if len(outcome.Steps) == 0 {+ outcome.Steps = current.Outcome.Steps+ }current.Outcome = outcomeif outcome.Status == st.Retrying || outcome.Status == st.Waiting {delay := 5 * time.Second * time.Duration(1<<min(run.AttemptCount-1, 6))@@ -384,36 +391,27 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st"currentOwner", conflictOwner,"currentStatus", conflictStatus,"outcomeStatus", outcome.Status)- return+ return nil}if err == nil {- return+ return nil}- slog.ErrorContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "error", err)- timer := time.NewTimer(time.Second)+ if attempt >= 10 {+ slog.ErrorContext(ctx, "Giving up persisting job result; releasing execution guard", "runId", run.ID, "attempts", attempt, "error", err)+ return fmt.Errorf("persist job result: %w", err)+ }+ slog.WarnContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "attempt", attempt, "retryIn", wait, "error", err)+ timer := time.NewTimer(wait)select {case <-ctx.Done():timer.Stop()- return+ return fmt.Errorf("persist job result: %w", err)case <-timer.C:}+ wait = min(wait*2, 30*time.Second)}}-func (q *Coordinator) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {- defer func() {- if recovered := recover(); recovered != nil {- outcome = st.Outcome{Status: st.Failed, Message: "Job panicked; inspect logs before retrying"}- err = fmt.Errorf("job panic: %v", recovered)- slog.ErrorContext(ctx, "Job panicked", "runId", run.ID, "error", err)- }- }()- if reconcile && q.reconcile != nil {- return q.reconcile(ctx, run)- }- return q.execute(ctx, run)-}-// Cancel marks an unstarted queued run canceled. Runs already attempted locally// or dispatched remotely must be reconciled rather than canceled as pending work.func (q *Coordinator) Cancel(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {@@ -435,11 +433,12 @@ func (q *Coordinator) Cancel(ctx context.Context, environmentID, jobID, runID stif err == nil {var state st.CoordinatorStateid := kit.SHA256Hex(environmentID + "\x00" + jobID)- actorType := executorTypeInternal- if strings.HasPrefix(jobID, "environment-health") || jobID == "docker-client-refresh" || jobID == "activity-sweep" {- actorType = healthExecutorTypeInternal- }- if readErr := q.service.GetState(ctx, coordinatorTypeInternal, id, &state); readErr == nil {+ actorType := kit.Ternary(+ strings.HasPrefix(jobID, "environment-health") || jobID == "docker-client-refresh" || jobID == "activity-sweep",+ healthExecutorType,+ executorType,+ )+ if readErr := q.service.GetState(ctx, coordinatorType, id, &state); readErr == nil {for _, intent := range state.Dispatches {if intent.RunID == runID && intent.JobID != "" {_ = q.service.DeleteJob(ctx, actorType, id, intent.JobID, actor.WithLiveJobsOnly())@@ -473,12 +472,12 @@ func (q *Coordinator) Retry(ctx context.Context, environmentID, jobID, runID strreturn nil})if err == nil {- q.signalInternal()+ q.signal()}return run, err}-func (q *Coordinator) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {+func (q *Coordinator) nextRun(runs []st.Run) (*st.Run, time.Time) {var interrupted *st.Runblocked := falsefor index := range runs {@@ -510,33 +509,13 @@ func (q *Coordinator) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {return nil, nextRetry}-func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {- for _, target := range previous.Targets {- found := false- for _, updated := range outcome.Targets {- if updated.ID == target.ID {- found = true- break- }- }- if !found {- outcome.Targets = append(outcome.Targets, target)- }- }- outcome.ActivityID = cmp.Or(outcome.ActivityID, previous.ActivityID)- if len(outcome.Steps) == 0 {- outcome.Steps = previous.Steps- }- return outcome-}-// ExecutionContext binds durable progress for work executing a run. An empty// instanceID fences writes by run owner; otherwise by the active workflow instance.func (q *Coordinator) ExecutionContext(ctx context.Context, run st.Run, instanceID string) context.Context {ctx = utils.WithActivityBatchID(ctx, run.ID)return jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {return q.UpdateRun(ctx, run, func(current *st.Run) error {- if !fencedInternal(current, run, instanceID) {+ if !kit.Ternary(instanceID == "", current.Status == st.Running && current.Owner == run.Owner, ActiveWorkflow(*current, instanceID)) {return ErrRunConflict}current.Outcome.Targets = jobcontext.Merge(current.Outcome.Targets, target)@@ -545,10 +524,10 @@ func (q *Coordinator) ExecutionContext(ctx context.Context, run st.Run, instance})}-// RecordSteps projects workflow step progress onto a running run.+// RecordSteps projects workflow step progress onto a running run, fenced like ExecutionContext.func (q *Coordinator) RecordSteps(ctx context.Context, run st.Run, instanceID string, steps []st.StepOutcome) error {return q.UpdateRun(ctx, run, func(current *st.Run) error {- if !fencedInternal(current, run, instanceID) {+ if !kit.Ternary(instanceID == "", current.Status == st.Running && current.Owner == run.Owner, ActiveWorkflow(*current, instanceID)) {return ErrRunConflict}current.Outcome.Steps = steps@@ -556,14 +535,6 @@ func (q *Coordinator) RecordSteps(ctx context.Context, run st.Run, instanceID st})}-// fencedInternal reports whether work may still write to a run.-func fencedInternal(current *st.Run, run st.Run, instanceID string) bool {- if instanceID == "" {- return current.Status == st.Running && current.Owner == run.Owner- }- return ActiveWorkflow(*current, instanceID)-}-// ActiveWorkflow reports whether a running run still follows a workflow instance.func ActiveWorkflow(run st.Run, instanceID string) bool {return run.Status == st.Running && slices.ContainsFunc(run.Outcome.Targets, func(target st.TargetOutcome) bool {@@ -602,7 +573,7 @@ func (q *Coordinator) Resolve(ctx context.Context, environmentID, jobID, runID,return nil})if err == nil {- q.signalInternal()+ q.signal()}return run, err}diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.goindex 18179dd47..6cbe0b069 100644--- a/backend/pkg/scheduler/runs/coordinator_store.go+++ b/backend/pkg/scheduler/runs/coordinator_store.go@@ -19,21 +19,21 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/kv")-const queuePrefixInternal = "jobs."+const queuePrefix = "jobs."-func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {+func (q *Coordinator) mutate(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {id := kit.SHA256Hex(environmentID + "\x00" + jobID)q.runtime.NameActor(id, jobID+"@"+environmentID)for range 20 {var state st.CoordinatorState- err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state)+ err := q.service.GetState(ctx, coordinatorType, id, &state)if err != nil && !errors.Is(err, actor.ErrStateNotFound) {return err}- if prepareMutationErr := q.prepareMutationInternal(&state, environmentID, jobID, change); prepareMutationErr != nil {+ if prepareMutationErr := q.prepareMutation(&state, environmentID, jobID, change); prepareMutationErr != nil {return prepareMutationErr}- result, err := q.service.Invoke(ctx, coordinatorTypeInternal, id, "replace", state)+ result, err := q.service.Invoke(ctx, coordinatorType, id, "replace", state)if err != nil {return err}@@ -48,10 +48,10 @@ func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID sreturn errors.New("job coordinator state is busy; retry mutation")}-func (q *Coordinator) prepareMutationInternal(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {- normalizeRecordTimesInternal(&state.Record)+func (q *Coordinator) prepareMutation(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {+ normalizeRecordTimes(&state.Record)previousRuns := slices.Clone(state.Record.Runs)- previous, versions, err := snapshotRunsInternal(state.Record)+ previous, versions, err := snapshotRuns(state.Record)if err != nil {return err}@@ -61,28 +61,28 @@ func (q *Coordinator) prepareMutationInternal(state *st.CoordinatorState, enviroif changeErr := change(&state.Record); changeErr != nil {return changeErr}- preserveUnsyncedRunsInternal(state, previousRuns)+ preserveUnsyncedRuns(state, previousRuns)for i := range state.Record.Runs {- q.associateActivityInternal(&state.Record.Runs[i])- if markChangedActivityErr := markChangedActivityInternal(state, &state.Record.Runs[i], previous, versions); markChangedActivityErr != nil {+ q.associateActivity(&state.Record.Runs[i])+ if markChangedActivityErr := markChangedActivity(state, &state.Record.Runs[i], previous, versions); markChangedActivityErr != nil {return markChangedActivityErr}}for id, run := range state.Record.Receipts {- q.associateActivityInternal(&run)- if markChangedActivityErr2 := markChangedActivityInternal(state, &run, previous, versions); markChangedActivityErr2 != nil {+ q.associateActivity(&run)+ if markChangedActivityErr2 := markChangedActivity(state, &run, previous, versions); markChangedActivityErr2 != nil {return markChangedActivityErr2}state.Record.Receipts[id] = run}- pruneDispatchesInternal(state)+ pruneDispatches(state)return nil}-func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]time.Time, error) {+func snapshotRuns(record st.QueueRecord) (map[string][]byte, map[string]time.Time, error) {encoded := make(map[string][]byte, len(record.Runs)+len(record.Receipts))versions := make(map[string]time.Time, len(record.Runs)+len(record.Receipts))- for _, run := range append(slices.Clone(record.Runs), receiptRunsInternal(record.Receipts)...) {+ for _, run := range append(slices.Clone(record.Runs), receiptRuns(record.Receipts)...) {data, err := json.Marshal(run)if err != nil {return nil, nil, err@@ -93,7 +93,7 @@ func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]return encoded, versions, nil}-func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run) {+func preserveUnsyncedRuns(state *st.CoordinatorState, previous []st.Run) {retained := make(map[string]st.Run, len(state.Record.Runs))for _, run := range state.Record.Runs {retained[run.ID] = run@@ -133,7 +133,7 @@ func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run)state.Record.Runs = runs}-func pruneDispatchesInternal(state *st.CoordinatorState) {+func pruneDispatches(state *st.CoordinatorState) {attempts := make(map[string]int, len(state.Record.Runs))for _, run := range state.Record.Runs {attempts[run.ID] = run.AttemptCount@@ -150,7 +150,7 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {records := []st.QueueRecord{}cursor := ""for {- page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})+ page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})if err != nil {return nil, fmt.Errorf("list job coordinator states: %w", err)}@@ -159,7 +159,7 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {if decodeErr := entry.Data.Decode(&state); decodeErr != nil {return nil, fmt.Errorf("decode job coordinator %q: %w", entry.ActorID, decodeErr)}- normalizeRecordTimesInternal(&state.Record)+ normalizeRecordTimes(&state.Record)q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)records = append(records, state.Record)}@@ -174,23 +174,23 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {// idempotency receipts. It returns ErrRunNotFound when no matching record exists.func (q *Coordinator) Get(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {var state st.CoordinatorState- err := q.service.GetState(ctx, coordinatorTypeInternal, kit.SHA256Hex(environmentID+"\x00"+jobID), &state)+ err := q.service.GetState(ctx, coordinatorType, kit.SHA256Hex(environmentID+"\x00"+jobID), &state)if errors.Is(err, actor.ErrStateNotFound) {return st.Run{}, ErrRunNotFound}if err != nil {return st.Run{}, err}- normalizeRecordTimesInternal(&state.Record)+ normalizeRecordTimes(&state.Record)record := state.Recordfor _, run := range record.Runs {if run.ID == runID {- q.associateActivityInternal(&run)+ q.associateActivity(&run)return run, nil}}if run, ok := record.Receipts[runID]; ok {- q.associateActivityInternal(&run)+ q.associateActivity(&run)return run, nil}return st.Run{}, ErrRunNotFound@@ -224,7 +224,7 @@ func (q *Coordinator) List(ctx context.Context, environmentID, jobID string, pag// and swap. The callback may run more than once and must not perform external side effects.func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run) error) error {wake := false- err := q.mutateInternal(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {+ err := q.mutate(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {for index := range record.Runs {if record.Runs[index].ID == run.ID {previous := record.Runs[index].Status@@ -248,26 +248,26 @@ func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*streturn ErrRunNotFound})if err == nil && wake {- q.signalInternal()+ q.signal()}if err == nil {- q.observeRunInternal(ctx, run)+ q.observeRun(ctx, run)}return err}-func (q *Coordinator) pruneInternal(ctx context.Context) error {+func (q *Coordinator) prune(ctx context.Context) error {records, err := q.Records(ctx)if err != nil {return err}now := time.Now().UTC()for _, record := range records {- if !pruneRunsInternal(&record, now) {+ if !pruneRuns(&record, now) {continue}- if mutateErr := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {- pruneRunsInternal(current, now)+ if mutateErr := q.mutate(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {+ pruneRuns(current, now)return nil}); mutateErr != nil {return fmt.Errorf("prune job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)@@ -276,7 +276,7 @@ func (q *Coordinator) pruneInternal(ctx context.Context) error {return nil}-func pruneRunsInternal(record *st.QueueRecord, now time.Time) bool {+func pruneRuns(record *st.QueueRecord, now time.Time) bool {changed := falseterminal := 0for index, run := range slices.Backward(record.Runs) {@@ -287,7 +287,7 @@ func pruneRunsInternal(record *st.QueueRecord, now time.Time) bool {if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {continue}- if separateTriggerInternal(run.Trigger) {+ if separateTrigger(run.Trigger) {if record.Receipts == nil {record.Receipts = make(map[string]st.Run)}@@ -321,14 +321,14 @@ func (q *Coordinator) SetExecutor(execute, reconcile func(context.Context, st.Rufunc (q *Coordinator) ScheduleState(ctx context.Context, jobID string) (st.QueueRecord, error) {var state st.CoordinatorState- err := q.service.GetState(ctx, coordinatorTypeInternal, kit.SHA256Hex("0\x00"+jobID), &state)- normalizeRecordTimesInternal(&state.Record)+ err := q.service.GetState(ctx, coordinatorType, kit.SHA256Hex("0\x00"+jobID), &state)+ normalizeRecordTimes(&state.Record)return state.Record, err}-func (q *Coordinator) Activate() { q.enabled.Store(true); q.signalInternal() }+func (q *Coordinator) Activate() { q.enabled.Store(true); q.signal() }-func receiptRunsInternal(receipts map[string]st.Run) []st.Run {+func receiptRuns(receipts map[string]st.Run) []st.Run {runs := make([]st.Run, 0, len(receipts))for _, run := range receipts {runs = append(runs, run)@@ -336,8 +336,8 @@ func receiptRunsInternal(receipts map[string]st.Run) []st.Run {return runs}-func markChangedActivityInternal(state *st.CoordinatorState, run *st.Run, previous map[string][]byte, previousUpdated map[string]time.Time) error {- normalizeRunTimesInternal(run)+func markChangedActivity(state *st.CoordinatorState, run *st.Run, previous map[string][]byte, previousUpdated map[string]time.Time) error {+ normalizeRunTimes(run)if run.ActivityID == "" {return nil}@@ -358,7 +358,7 @@ func markChangedActivityInternal(state *st.CoordinatorState, run *st.Run, previoreturn nil}-func markActivitySyncInternal(state *st.CoordinatorState, run st.Run) {+func markActivitySync(state *st.CoordinatorState, run st.Run) {if run.ActivityID == "" {return}@@ -368,35 +368,35 @@ func markActivitySyncInternal(state *st.CoordinatorState, run st.Run) {state.PendingActivitySync[run.ID] = run.UpdatedAt}-func normalizeRecordTimesInternal(record *st.QueueRecord) {+func normalizeRecordTimes(record *st.QueueRecord) {record.LastEnqueuedAt = record.LastEnqueuedAt.UTC()record.NextRun = record.NextRun.UTC()for i := range record.Runs {- normalizeRunTimesInternal(&record.Runs[i])+ normalizeRunTimes(&record.Runs[i])}for id, run := range record.Receipts {- normalizeRunTimesInternal(&run)+ normalizeRunTimes(&run)record.Receipts[id] = run}}-func normalizeRunTimesInternal(run *st.Run) {+func normalizeRunTimes(run *st.Run) {run.CreatedAt = run.CreatedAt.UTC()run.UpdatedAt = run.UpdatedAt.UTC()- normalizeTimePointerInternal(run.StartedAt)- normalizeTimePointerInternal(run.FinishedAt)- normalizeTimePointerInternal(run.NextAttempt)- normalizeTimePointerInternal(run.LastConfirmedAt)+ normalizeTimePointer(run.StartedAt)+ normalizeTimePointer(run.FinishedAt)+ normalizeTimePointer(run.NextAttempt)+ normalizeTimePointer(run.LastConfirmedAt)if run.Resolution != nil {run.Resolution.ResolvedAt = run.Resolution.ResolvedAt.UTC()}for i := range run.Attempts {run.Attempts[i].StartedAt = run.Attempts[i].StartedAt.UTC()- normalizeTimePointerInternal(run.Attempts[i].FinishedAt)+ normalizeTimePointer(run.Attempts[i].FinishedAt)}}-func normalizeLegacyRunInternal(run *st.Run) bool {+func normalizeLegacyRun(run *st.Run) bool {automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == user.SystemUser.Usernameif run.Status != st.NeedsAttention && !automaticResolution {return false@@ -417,7 +417,7 @@ func normalizeLegacyRunInternal(run *st.Run) bool {return true}-func normalizeTimePointerInternal(value *time.Time) {+func normalizeTimePointer(value *time.Time) {if value != nil {*value = value.UTC()}diff --git a/backend/pkg/scheduler/runs/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.goindex a8d5fc078..5b7f3b062 100644--- a/backend/pkg/scheduler/runs/resolution_test.go+++ b/backend/pkg/scheduler/runs/resolution_test.go@@ -15,7 +15,7 @@ import (francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func newResolutionQueueInternal(t *testing.T) *Coordinator {+func newResolutionQueue(t *testing.T) *Coordinator {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)@@ -32,7 +32,7 @@ func newResolutionQueueInternal(t *testing.T) *Coordinator {}func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)ctx := t.Context()run, err := q.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "manual"})require.NoError(t, err)@@ -48,7 +48,7 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {require.NoError(t, err)history, err := q.List(ctx, "0", "auto-update", 1, 20)require.NoError(t, err)- selected, _ := q.nextRunInternal(history.Runs)+ selected, _ := q.nextRun(history.Runs)require.NotNil(t, selected)require.Equal(t, pending.ID, selected.ID)for len(q.wake) > 0 {@@ -69,7 +69,7 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {}history, err = q.List(ctx, "0", "auto-update", 1, 20)require.NoError(t, err)- selected, _ = q.nextRunInternal(history.Runs)+ selected, _ = q.nextRun(history.Runs)require.NotNil(t, selected)require.Equal(t, pending.ID, selected.ID)duplicate, err := q.Resolve(ctx, "0", "auto-update", run.ID, "different operator")@@ -139,7 +139,7 @@ func TestResolveRejectsUnsafeStates(t *testing.T) {{name: "uncertain remote", status: st.NeedsAttention, environment: "remote", delivered: true},} {t.Run(test.name, func(t *testing.T) {- q := newResolutionQueueInternal(t)+ q := newResolutionQueue(t)run, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: test.environment})require.NoError(t, err)require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error {diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.goindex aaad6b958..8c6c49243 100644--- a/backend/pkg/utils/francis/provider.go+++ b/backend/pkg/utils/francis/provider.go@@ -66,7 +66,7 @@ func StoreURL(databaseURL string) (string, error) {// The sibling file must be creatable and writable whatever mode opens arcane.db.query.Del("mode")if !sqliteutil.HasPragma(query, "busy_timeout") {- query.Add("_pragma", "busy_timeout(2500)")+ query.Add("_pragma", "busy_timeout(10000)")}// NORMAL stops WAL commits from holding the write lock through an fsync.if !sqliteutil.HasPragma(query, "synchronous") {diff --git a/backend/pkg/utils/francis/provider_test.go b/backend/pkg/utils/francis/provider_test.goindex 1b2a65b7e..dd5ecef1a 100644--- a/backend/pkg/utils/francis/provider_test.go+++ b/backend/pkg/utils/francis/provider_test.go@@ -18,7 +18,7 @@ func TestMigrateLegacyStore(t *testing.T) {databaseURL := "file:" + mainPath + "?_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate"storeURL, err := StoreURL(databaseURL)require.NoError(t, err)- const defaults = "_pragma=busy_timeout%282500%29&_pragma=synchronous%28NORMAL%29"+ const defaults = "_pragma=busy_timeout%2810000%29&_pragma=synchronous%28NORMAL%29"require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+defaults+"&_txlock=immediate", storeURL)for input, expected := range map[string]string{"file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db?" + defaults + "&_txlock=immediate",@@ -26,8 +26,8 @@ func TestMigrateLegacyStore(t *testing.T) {"file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db?" + defaults,"file:data/arcane": "file:data/arcane.francis.db?" + defaults,"file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29",- "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%282500%29",- "file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%282500%29",+ "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%2810000%29",+ "file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%2810000%29","postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",} {storeURL, err = StoreURL(input)diff --git a/backend/pkg/utils/imageref/repodigest.go b/backend/pkg/utils/imageref/repodigest.gonew file mode 100644index 000000000..ef589e3af--- /dev/null+++ b/backend/pkg/utils/imageref/repodigest.go@@ -0,0 +1,14 @@+package imageref++import "strings"++// RepoDigestRepository returns the repository of the first usable+// "repo@digest" entry in an image's RepoDigests.+func RepoDigestRepository(repoDigests []string) (string, bool) {+ for _, repoDigest := range repoDigests {+ if repository, _, found := strings.CutLast(repoDigest, "@"); repoDigest != "<none>@<none>" && found && repository != "" {+ return repository, true+ }+ }+ return "", false+}diff --git a/backend/pkg/utils/imageref/repodigest_test.go b/backend/pkg/utils/imageref/repodigest_test.gonew file mode 100644index 000000000..1c4a53001--- /dev/null+++ b/backend/pkg/utils/imageref/repodigest_test.go@@ -0,0 +1,38 @@+package imageref++import "testing"++func TestRepoDigestRepository(t *testing.T) {+ tests := []struct {+ name string+ repoDigests []string+ want string+ wantOK bool+ }{+ {name: "nil"},+ {name: "none placeholder only", repoDigests: []string{"<none>@<none>"}},+ {name: "missing at sign", repoDigests: []string{"nginx"}},+ {name: "empty repository", repoDigests: []string{"@sha256:abc"}},+ {+ name: "first usable entry wins",+ repoDigests: []string{"<none>@<none>", "ghcr.io/getarcaneapp/arcane@sha256:abc", "nginx@sha256:def"},+ want: "ghcr.io/getarcaneapp/arcane",+ wantOK: true,+ },+ {+ name: "registry with port",+ repoDigests: []string{"localhost:5000/app@sha256:abc"},+ want: "localhost:5000/app",+ wantOK: true,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, ok := RepoDigestRepository(tt.repoDigests)+ if got != tt.want || ok != tt.wantOK {+ t.Fatalf("RepoDigestRepository(%q) = (%q, %v), want (%q, %v)", tt.repoDigests, got, ok, tt.want, tt.wantOK)+ }+ })+ }+}diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yamlindex ab736a43d..3dcb749e5 100644--- a/docker/compose.dev.yaml+++ b/docker/compose.dev.yaml@@ -60,11 +60,12 @@ services:- PGID=1000- ENCRYPTION_KEY=dev-encryption-key-replace-in-production-must-be-32-chars- JWT_SECRET=dev-jwt-secret-replace-in-production-must-be-long-enough- - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate+ - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediateworking_dir: /app/backendnetworks:- arcane-devrestart: unless-stopped+ stop_grace_period: 15shealthcheck:test: ['CMD-SHELL', 'curl -fsS http://localhost:3552/api/health >/dev/null || exit 1']interval: 30s@@ -92,7 +93,7 @@ services:- EDGE_TRANSPORT=websocket- ENCRYPTION_KEY=dev-encryption-key-replace-in-production-must-be-32-chars- JWT_SECRET=dev-jwt-secret-replace-in-production-must-be-long-enough- - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate+ - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediatevolumes:- /var/run/docker.sock:/var/run/docker.sock- agent-dev-data:/app/data@@ -108,6 +109,7 @@ services:networks:- arcane-devrestart: unless-stopped+ stop_grace_period: 15shealthcheck:test: ['CMD-SHELL', 'curl -fsS http://localhost:3553/api/health >/dev/null || exit 1']interval: 30sdiff --git a/docker/examples/compose.agent-edge.yaml b/docker/examples/compose.agent-edge.yamlindex 75aefba82..4bc94dd16 100644--- a/docker/examples/compose.agent-edge.yaml+++ b/docker/examples/compose.agent-edge.yaml@@ -1,6 +1,6 @@services:arcane-edge-agent:- image: ghcr.io/getarcaneapp/agent:latest+ image: getarcaneapp/agent:latestcontainer_name: arcane-edge-agent# No ports exposed - agent connects outbound to manager tunnel transportenvironment:@@ -20,6 +20,7 @@ services:# This allows Arcane to detect its own container ID more reliablycgroup: hostrestart: unless-stopped+ stop_grace_period: 15svolumes:agent-data:diff --git a/docker/examples/compose.agent.yaml b/docker/examples/compose.agent.yamlindex 6dd4e6568..83c8caaec 100644--- a/docker/examples/compose.agent.yaml+++ b/docker/examples/compose.agent.yaml@@ -1,6 +1,6 @@services:arcane-agent:- image: ghcr.io/getarcaneapp/agent:latest+ image: getarcaneapp/agent:latestcontainer_name: arcane-agentports:- '3553:3553'@@ -21,6 +21,7 @@ services:# This allows Arcane to detect its own container ID more reliablycgroup: hostrestart: unless-stopped+ stop_grace_period: 15svolumes:agent-data:diff --git a/docker/examples/compose.basic.yaml b/docker/examples/compose.basic.yamlindex d864c8a12..76f04313a 100644--- a/docker/examples/compose.basic.yaml+++ b/docker/examples/compose.basic.yaml@@ -1,6 +1,6 @@services:arcane:- image: ghcr.io/getarcaneapp/manager:latest+ image: getarcaneapp/manager:latestcontainer_name: arcaneports:- '3552:3552'@@ -29,6 +29,7 @@ services:retries: 5start_period: 15srestart: unless-stopped+ stop_grace_period: 15svolumes:arcane-data:diff --git a/docker/examples/compose.otel.yaml b/docker/examples/compose.otel.yamlindex 10f2e94ae..0ddf7f4ea 100644--- a/docker/examples/compose.otel.yaml+++ b/docker/examples/compose.otel.yaml@@ -1,6 +1,6 @@services:arcane:- image: ghcr.io/getarcaneapp/manager:latest+ image: getarcaneapp/manager:latestcontainer_name: arcaneports:- '3552:3552'@@ -20,6 +20,7 @@ services:depends_on:- otel-collectorrestart: unless-stopped+ stop_grace_period: 15sotel-collector:image: otel/opentelemetry-collector-contrib:latestdiff --git a/docker/examples/compose.proxy.yaml b/docker/examples/compose.proxy.yamlindex 14444c5c4..c64d3464c 100644--- a/docker/examples/compose.proxy.yaml+++ b/docker/examples/compose.proxy.yaml@@ -38,7 +38,7 @@ services:- no-new-privileges:truearcane:- image: ghcr.io/getarcaneapp/arcane:latest+ image: getarcaneapp/arcane:latestcontainer_name: arcaneports:- '3552:3552'@@ -65,6 +65,7 @@ services:# This allows Arcane to detect its own container ID more reliablycgroup: hostrestart: unless-stopped+ stop_grace_period: 15snetworks:arcane-internal:diff --git a/just/dev.just b/just/dev.justindex f92303cd2..7242353a8 100644--- a/just/dev.just+++ b/just/dev.just@@ -28,7 +28,7 @@ agent:manager_api_url="${MANAGER_API_URL:-https://localhost:3552}"edge_mtls_assets_dir="${EDGE_MTLS_ASSETS_DIR:-./.tmp/edge-test-agent/edge-mtls-agent}"edge_mtls_ca_file="${EDGE_MTLS_CA_FILE:-./backend/local-manager.crt}"- database_url="${DATABASE_URL:-file:./.tmp/edge-test-agent/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate}"+ database_url="${DATABASE_URL:-file:./.tmp/edge-test-agent/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate}"projects_directory="${PROJECTS_DIRECTORY:-./.tmp/edge-test-agent/projects}"git_work_dir="${GIT_WORK_DIR:-./.tmp/edge-test-agent/git}"jwt_secret="${JWT_SECRET:-local-edge-test-jwt-secret-please-change}"diff --git a/types/system/health.go b/types/system/health.goindex 07df4b0ec..a7bf4de9b 100644--- a/types/system/health.go+++ b/types/system/health.go@@ -7,3 +7,15 @@ type HealthResponse struct {// Required: trueStatus string `json:"status"`}++// ReadinessResponse is the body of the readiness probe; Checks maps each dependency to "ok" or its error.+type ReadinessResponse struct {+ // Status is "ready" or "not_ready".+ //+ // Required: true+ Status string `json:"status"`+ // Checks maps each dependency name to "ok" or its error.+ //+ // Required: true+ Checks map[string]string `json:"checks"`+}