Skip to content

Modern Docker Management, Designed for Everyone

refactor: rework the Docker boundary, image ownership, and other logic (#4484)

Kyle Mendellfd742f3ee594
diff --git a/.env.example b/.env.example
index 37c103e34..0d6177d5c 100644
--- a/.env.example
+++ b/.env.example
@@ -42,7 +42,7 @@ JWT_SECRET=your-super-secret-jwt-key-change-this
# JWT_REFRESH_EXPIRY=168h
# Database Configuration
-DATABASE_URL=file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate
+DATABASE_URL=file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate
# Downgrades are blocked by default; enable only temporarily after taking a backup.
ALLOW_DOWNGRADE=false
# Optional: GitHub API token used for downgrade migration fetches to avoid rate limits.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 8e2b67dbc..228208473 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -290,8 +290,11 @@ When `./scripts/development/dev.sh start` fails or the app does not load, valida
```bash
curl -f http://localhost:3000
curl -f http://localhost:3552/api/health
+ curl -f http://localhost:3552/api/health/ready
```
+ `/api/health` reports process liveness. `/api/health/ready` also pings the database and Docker and returns 503 until both answer; use it for orchestrator readiness probes if you need one.
+
5. If a service is unhealthy, read the targeted logs first:
```bash
diff --git a/backend/api/api.go b/backend/api/api.go
index 74124264e..4722c625a 100644
--- a/backend/api/api.go
+++ b/backend/api/api.go
@@ -61,7 +61,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/userctx"
)
@@ -227,7 +226,7 @@ type HandlerDeps struct {
Role *role.Module
Variable *variable.Module
Upload *upload.Module
- Actors *francis.Runtime
+ Health *health.Module
}
// SetupAPI creates and configures the Huma API attached to the Echo router.
@@ -274,7 +273,7 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApp
api.UseMiddleware(middleware.NewActivityBatchID())
registerNormalization(api)
- health.RegisterRoutes(api, deps.Actors)
+ deps.Health.RegisterRoutes(api)
deps.Auth.RegisterRoutes(api)
passkey.RegisterPasskeys(api, deps.Passkey, deps.Auth.Service(), deps.User.Service())
deps.ApiKey.RegisterRoutes(api)
@@ -328,5 +327,5 @@ func SetupAPI(e *echo.Echo, apiGroup *echo.Group, appCtx handlerutil.ActivityApp
func SetupAPIForSpec() huma.API {
e := echo.New()
appCtx := handlerutil.NewActivityAppContext(context.Background()) //nolint:forbidigo // Schema discovery has no running request or application lifecycle.
- return SetupAPI(e, e.Group("/api"), appCtx, &config.Config{}, HandlerDeps{})
+ return SetupAPI(e, e.Group("/api"), appCtx, &config.Config{}, HandlerDeps{Health: health.New(&health.Service{})})
}
diff --git a/backend/api/api_test.go b/backend/api/api_test.go
index 194782de8..0e78de956 100644
--- a/backend/api/api_test.go
+++ b/backend/api/api_test.go
@@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/health"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
@@ -342,7 +343,7 @@ func TestVariableMaterializationRoutesAreAgentOnly(t *testing.T) {
router.Group("/api"),
handlerutil.NewActivityAppContext(t.Context()),
&config.Config{AgentMode: true},
- HandlerDeps{},
+ HandlerDeps{Health: health.New(&health.Service{})},
)
require.Nil(t, agentAPI.OpenAPI().Paths["/variables"])
materialized := agentAPI.OpenAPI().Paths["/environments/{id}/templates/variables"]
diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.go
index 3387207ff..d6b360902 100644
--- a/backend/api/ws/container_stats.go
+++ b/backend/api/ws/container_stats.go
@@ -42,10 +42,14 @@ func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {
// A reconnect can land exactly on the 5s idle-teardown boundary and load a
// hub whose Run has already exited. Drop that hub and retry once so the
// client gets a live producer instead of a silent socket.
- // The hub owns connection cleanup; preserve request values without request cancellation.
+ // The hub owns connection cleanup, so it outlives the request and ends with the app.
for attempt := range 2 {
- hub := h.getOrCreateContainerStatsHub(containerID)
- if wshub.ServeClientWithOnRemove(context.WithoutCancel(c.Request().Context()), hub, conn, onRemove) {
+ actual, loaded := h.containerStatsHubs.LoadOrStore(containerID, wshub.NewHub(64))
+ hub, _ := actual.(*wshub.Hub)
+ if !loaded {
+ h.runContainerStatsHub(containerID, hub)
+ }
+ if wshub.ServeClientWithOnRemove(h.ctx, hub, conn, onRemove) {
return nil
}
h.containerStatsHubs.CompareAndDelete(containerID, hub)
@@ -59,34 +63,12 @@ func (h *WebSocketHandler) ContainerStats(c *echo.Context) error {
return nil
}
-func (h *WebSocketHandler) getOrCreateContainerStatsHub(containerID string) *wshub.Hub {
- if existing, ok := h.containerStatsHubs.Load(containerID); ok {
- if hub, localOk := existing.(*wshub.Hub); localOk {
- return hub
- }
- }
-
- hub := wshub.NewHub(64)
- actual, loaded := h.containerStatsHubs.LoadOrStore(containerID, hub)
- if loaded {
- if existingHub, ok := actual.(*wshub.Hub); ok {
- return existingHub
- }
- // type assertion failure is impossible in practice, but avoid running
- // an unregistered hub if it somehow occurs
- return hub
- }
-
- h.runContainerStatsHub(containerID, hub)
- return hub
-}
-
// containerStatsErrorDrainGrace is how long a failed stats hub stays alive after
// broadcasting its error frame, so clients receive it before the hub shuts down.
const containerStatsErrorDrainGrace = 2 * time.Second
func (h *WebSocketHandler) runContainerStatsHub(containerID string, hub *wshub.Hub) {
- ctx, cancel := context.WithCancel(context.Background()) //nolint:forbidigo // Shared stats hubs own their lifecycle and idle cancellation across requests.
+ ctx, cancel := context.WithCancel(h.ctx)
var cleanupTimer *time.Timer
var cleanupTimerMu sync.Mutex
diff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.go
index 7c1f20e15..8e29180d8 100644
--- a/backend/api/ws/handler.go
+++ b/backend/api/ws/handler.go
@@ -41,6 +41,9 @@ var (
// WebSocketHandler consolidates all WebSocket and streaming endpoints.
// REST endpoints are handled by Huma handlers.
type WebSocketHandler struct {
+ // ctx ends after open connections were closed with StatusGoingAway on app shutdown.
+ ctx context.Context
+ conns sync.Map
projectService *project.ProjectService
containerService *container.ContainerService
swarmService *swarm.SwarmService
@@ -80,6 +83,7 @@ type WebSocketHandler struct {
}
func NewWebSocketHandler(
+ appCtx context.Context,
group *echo.Group,
projectService *project.ProjectService,
containerService *container.ContainerService,
@@ -89,7 +93,9 @@ func NewWebSocketHandler(
authMiddleware *auth.AuthMiddleware,
cfg *config.Config,
) {
+ ctx, cancel := context.WithCancel(context.WithoutCancel(appCtx))
handler := &WebSocketHandler{
+ ctx: ctx,
projectService: projectService,
containerService: containerService,
swarmService: swarmService,
@@ -103,6 +109,18 @@ func NewWebSocketHandler(
Build(),
checkWSOrigin: httpx.ValidateWebSocketOrigin(cfg.GetAppURL()),
}
+ // Send the close frames before cancelling hubs, whose teardown closes connections without a handshake.
+ context.AfterFunc(appCtx, func() {
+ var wg sync.WaitGroup
+ handler.conns.Range(func(key, _ any) bool {
+ wg.Go(func() {
+ _ = key.(*websocket.Conn).Close(websocket.StatusGoingAway, "server shutting down")
+ })
+ return true
+ })
+ wg.Wait()
+ cancel()
+ })
observeWebSocketMetricsOnce.Do(func() {
if err := defaultWebSocketMetrics.ObserveConnections(otel.Meter(tracing.InstrumentationName)); err != nil {
otel.Handle(err)
@@ -134,6 +152,7 @@ func (h *WebSocketHandler) acceptWS(c *echo.Context, kind, resourceID string) (*
UserAgent: req.Header.Get("User-Agent"),
}
connID := defaultWebSocketMetrics.RegisterConnection(info)
+ h.conns.Store(conn, struct{}{})
// The HTTP tracing middleware skips upgrades, so this is a root span unless the client sent a traceparent.
parent := otel.GetTextMapPropagator().Extract(req.Context(), propagation.HeaderCarrier(req.Header))
_, span := otel.Tracer(tracing.InstrumentationName).Start(parent, "websocket "+kind,
@@ -149,6 +168,7 @@ func (h *WebSocketHandler) acceptWS(c *echo.Context, kind, resourceID string) (*
attribute.String("user_agent.original", info.UserAgent),
))
return conn, func() {
+ h.conns.Delete(conn)
defaultWebSocketMetrics.UnregisterConnection(connID)
span.End()
}, true
diff --git a/backend/api/ws/log_streams.go b/backend/api/ws/log_streams.go
index f40101adb..67db95130 100644
--- a/backend/api/ws/log_streams.go
+++ b/backend/api/ws/log_streams.go
@@ -33,20 +33,6 @@ type wsLogStream struct {
seq atomic.Uint64
}
-func buildLogStreamKey(envID, kind, resourceID, format string, batched, follow bool, tail, since string, timestamps bool) string {
- return strings.Join([]string{
- envID,
- kind,
- resourceID,
- format,
- strconv.FormatBool(batched),
- strconv.FormatBool(follow),
- tail,
- since,
- strconv.FormatBool(timestamps),
- }, "|")
-}
-
func (h *WebSocketHandler) getOrCreateLogStream(key string, create func(onEmpty func(*wsLogStream)) *wsLogStream) *wsLogStream {
h.logStreamsMu.Lock()
defer h.logStreamsMu.Unlock()
@@ -59,7 +45,7 @@ func (h *WebSocketHandler) getOrCreateLogStream(key string, create func(onEmpty
}
stream := create(func(stream *wsLogStream) {
- h.markLogStreamDone(key, stream)
+ h.releaseLogStream(key, stream, true)
})
stream.key = key
stream.refs = 1
@@ -67,42 +53,22 @@ func (h *WebSocketHandler) getOrCreateLogStream(key string, create func(onEmpty
return stream
}
-func takeLogStreamCancel(stream *wsLogStream) context.CancelFunc {
- cancel := stream.cancel
- stream.cancel = nil
- return cancel
-}
-
-func (h *WebSocketHandler) releaseLogStream(key string, stream *wsLogStream) {
+// releaseLogStream drops a client reference, or marks the source finished when done is set,
+// and tears the stream down once no clients remain.
+func (h *WebSocketHandler) releaseLogStream(key string, stream *wsLogStream, done bool) {
var cancel context.CancelFunc
h.logStreamsMu.Lock()
- if stream.refs > 0 {
- stream.refs--
+ if done {
+ stream.done = true
+ } else {
+ stream.refs = max(stream.refs-1, 0)
}
if stream.refs == 0 {
if current, ok := h.logStreams[key]; ok && current == stream {
delete(h.logStreams, key)
}
- cancel = takeLogStreamCancel(stream)
- }
- h.logStreamsMu.Unlock()
-
- if cancel != nil {
- cancel()
- }
-}
-
-func (h *WebSocketHandler) markLogStreamDone(key string, stream *wsLogStream) {
- var cancel context.CancelFunc
-
- h.logStreamsMu.Lock()
- stream.done = true
- if stream.refs == 0 {
- if current, ok := h.logStreams[key]; ok && current == stream {
- delete(h.logStreams, key)
- }
- cancel = takeLogStreamCancel(stream)
+ cancel, stream.cancel = stream.cancel, nil
}
h.logStreamsMu.Unlock()
@@ -147,16 +113,26 @@ func (h *WebSocketHandler) serveLogStream(
return
}
- streamKey := buildLogStreamKey(c.Param("id"), kind, resourceID, params.format, params.batched, params.follow, params.tail, params.since, params.timestamps)
+ streamKey := strings.Join([]string{
+ c.Param("id"),
+ kind,
+ resourceID,
+ params.format,
+ strconv.FormatBool(params.batched),
+ strconv.FormatBool(params.follow),
+ params.tail,
+ params.since,
+ strconv.FormatBool(params.timestamps),
+ }, "|")
stream := h.getOrCreateLogStream(streamKey, func(onEmpty func(*wsLogStream)) *wsLogStream {
return hubBuilder(streamKey, onEmpty)
})
release := func() {
unregister()
- h.releaseLogStream(streamKey, stream)
+ h.releaseLogStream(streamKey, stream, false)
}
- // The hub owns connection cleanup; preserve request values without request cancellation.
- if !wshub.ServeClientWithOnRemove(context.WithoutCancel(c.Request().Context()), stream.hub, conn, release) {
+ // The hub owns connection cleanup, so it outlives the request and ends with the app.
+ if !wshub.ServeClientWithOnRemove(h.ctx, stream.hub, conn, release) {
// The stream refcount normally keeps this hub alive, so a stopped hub
// here means it was torn down out from under us; drop our reference
// rather than leaking the connection and the metrics entry.
@@ -226,43 +202,25 @@ func (h *WebSocketHandler) ProjectLogs(c *echo.Context) error {
"project",
params,
streamLogs,
- normalizeProjectLogMessage,
- normalizeProjectLogText,
+ func(line string) wshub.LogMessage {
+ level, service, message, timestamp := wshub.NormalizeProjectLine(line)
+ return wshub.LogMessage{
+ Level: level,
+ Message: message,
+ Service: service,
+ Timestamp: timestamp,
+ }
+ },
+ func(line string) string {
+ _, _, message, _ := wshub.NormalizeProjectLine(line)
+ return message
+ },
onEmpty,
)
})
return nil
}
-func newWSLogStream(key, format string) (*wsLogStream, context.Context) {
- ls := &wsLogStream{
- hub: wshub.NewHub(1024),
- firstSubscriber: make(chan struct{}),
- format: format,
- key: key,
- }
- ls.hub.SetOnFirstClient(func() {
- close(ls.firstSubscriber)
- })
-
- ctx, cancel := context.WithCancel(context.Background()) //nolint:gosec,forbidigo // Shared log hubs own their lifecycle and idle cancellation across requests.
- ls.cancel = cancel
-
- go ls.hub.Run(ctx)
-
- return ls, ctx
-}
-
-func normalizeProjectLogMessage(line string) wshub.LogMessage {
- level, service, message, timestamp := wshub.NormalizeProjectLine(line)
- return wshub.LogMessage{
- Level: level,
- Message: message,
- Service: service,
- Timestamp: timestamp,
- }
-}
-
func normalizeContainerLogMessage(line string) wshub.LogMessage {
level, message, timestamp := wshub.NormalizeContainerLine(line)
return wshub.LogMessage{
@@ -272,11 +230,6 @@ func normalizeContainerLogMessage(line string) wshub.LogMessage {
}
}
-func normalizeProjectLogText(line string) string {
- _, _, message, _ := wshub.NormalizeProjectLine(line)
- return message
-}
-
func (h *WebSocketHandler) startLogHub(
key, resourceID, label string,
params logStreamParams,
@@ -285,7 +238,18 @@ func (h *WebSocketHandler) startLogHub(
normalizeText func(string) string,
onEmptyHook func(*wsLogStream),
) *wsLogStream {
- ls, ctx := newWSLogStream(key, params.format)
+ ctx, cancel := context.WithCancel(h.ctx) //nolint:gosec // Cancel is stored on the stream and runs on idle teardown.
+ ls := &wsLogStream{
+ hub: wshub.NewHub(1024),
+ firstSubscriber: make(chan struct{}),
+ format: params.format,
+ key: key,
+ cancel: cancel,
+ }
+ ls.hub.SetOnFirstClient(func() {
+ close(ls.firstSubscriber)
+ })
+ go ls.hub.Run(ctx)
ls.hub.SetOnEmpty(func() {
if onEmptyHook != nil {
@@ -322,13 +286,13 @@ func (h *WebSocketHandler) startLogSource(
return
}
- h.markLogStreamDone(key, ls)
+ h.releaseLogStream(key, ls, true)
broadcastLogStreamError(ctx, label, resourceID, params.format, err, ls)
return
}
if ctx.Err() == nil {
- h.markLogStreamDone(key, ls)
+ h.releaseLogStream(key, ls, true)
}
}()
@@ -346,11 +310,14 @@ func startLogForwarders(
if params.format == "json" {
messages := mapLogLines(ctx, lines, func(line string) wshub.LogMessage {
message := normalizeJSON(line)
- message.Seq = ls.seq.Add(1)
- if message.Timestamp == "" {
- message.Timestamp = wshub.NowRFC3339()
+ return wshub.LogMessage{
+ Seq: ls.seq.Add(1),
+ Level: message.Level,
+ Message: message.Message,
+ Timestamp: cmp.Or(message.Timestamp, wshub.NowRFC3339()),
+ Service: message.Service,
+ ContainerID: message.ContainerID,
}
- return message
})
if params.batched {
diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.go
index 211c68e8e..127e43b0f 100644
--- a/backend/api/ws/system_stats.go
+++ b/backend/api/ws/system_stats.go
@@ -12,7 +12,6 @@ import (
"github.com/coder/websocket"
systemtypes "github.com/getarcaneapp/arcane/types/v2/system"
"github.com/labstack/echo/v5"
- "github.com/samber/hot"
"github.com/shirou/gopsutil/v4/cpu"
"github.com/shirou/gopsutil/v4/disk"
"github.com/shirou/gopsutil/v4/host"
@@ -37,31 +36,6 @@ const (
// System WebSocket Endpoints
// ============================================================================
-// checkRateLimit checks and applies rate limiting for WebSocket connections.
-func (h *WebSocketHandler) checkRateLimit(clientIP string) bool {
- h.activeConnectionsMu.Lock()
- defer h.activeConnectionsMu.Unlock()
- if h.activeConnections[clientIP] >= 5 {
- return false
- }
- if h.activeConnections == nil {
- h.activeConnections = make(map[string]int)
- }
- h.activeConnections[clientIP]++
- return true
-}
-
-// releaseRateLimit decrements the connection counter and cleans up if needed.
-func (h *WebSocketHandler) releaseRateLimit(clientIP string) {
- h.activeConnectionsMu.Lock()
- defer h.activeConnectionsMu.Unlock()
- if count := h.activeConnections[clientIP]; count > 1 {
- h.activeConnections[clientIP] = count - 1
- } else {
- delete(h.activeConnections, clientIP)
- }
-}
-
func (h *WebSocketHandler) acquireSystemStatsSampler(ctx context.Context, interval time.Duration) bool {
h.systemStatsSampler.lifecycleMu.Lock()
@@ -77,7 +51,7 @@ func (h *WebSocketHandler) acquireSystemStatsSampler(ctx context.Context, interv
return waitForSystemStatsSamplerReady(ctx, ready)
}
- samplerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
+ samplerCtx, cancel := context.WithCancel(h.ctx)
ready := make(chan struct{})
wake := make(chan struct{}, 1)
h.systemStatsSampler.cancel = cancel
@@ -206,7 +180,9 @@ func (h *WebSocketHandler) runSystemStatsSampler(ctx context.Context, wake <-cha
case <-wake:
rearm()
case <-ticker.C:
- h.updateCPUCache(0)
+ if val, ok := h.readCPUUsage(0); ok {
+ h.cpuCache.Store(val)
+ }
h.systemStatsSampler.latest.Store(collect(ctx))
rearm()
}
@@ -217,73 +193,62 @@ func (h *WebSocketHandler) runSystemStatsSampler(ctx context.Context, wake <-cha
func (h *WebSocketHandler) collectSystemStats(ctx context.Context) systemtypes.SystemStats {
cpuUsage, _ := h.cpuCache.Load()
- cpuCount := h.getCPUCount()
- memUsed, memTotal := h.getMemoryInfo()
- cpuCount, memUsed, memTotal = h.applyCgroupLimits(ctx, cpuCount, memUsed, memTotal)
- diskUsed, diskTotal := h.getDiskInfo(ctx)
- hostname := h.getHostname()
- gpuStats, gpuCount := h.getGPUInfo(ctx)
-
- return systemtypes.SystemStats{
- CPUUsage: cpuUsage,
- MemoryUsage: memUsed,
- MemoryTotal: memTotal,
- DiskUsage: diskUsed,
- DiskTotal: diskTotal,
- CPUCount: cpuCount,
- Architecture: runtime.GOARCH,
- Platform: runtime.GOOS,
- Hostname: hostname,
- GPUCount: gpuCount,
- GPUs: gpuStats,
- }
-}
-
-// getCPUCount returns the number of CPUs.
-func (h *WebSocketHandler) getCPUCount() int {
- h.initSystemStaticInfo()
- return h.systemStaticInfo.cpuCount
-}
-
-func (h *WebSocketHandler) initSystemStaticInfo() {
h.systemStaticInfo.once.Do(func() {
cpuCount, err := cpu.Counts(true)
if err != nil {
cpuCount = runtime.NumCPU()
}
-
hostInfo, _ := host.Info()
- hostname := ""
- if hostInfo != nil {
- hostname = hostInfo.Hostname
- }
-
h.systemStaticInfo.cpuCount = cpuCount
- h.systemStaticInfo.hostname = hostname
+ h.systemStaticInfo.hostname = kit.FromPtr(hostInfo).Hostname
})
-}
-// getMemoryInfo returns memory usage and total.
-func (h *WebSocketHandler) getMemoryInfo() (uint64, uint64) {
- memInfo, _ := mem.VirtualMemory()
- if memInfo == nil {
- return 0, 0
+ var memUsed, memTotal uint64
+ if memInfo, _ := mem.VirtualMemory(); memInfo != nil {
+ // gopsutil counts ZFS ARC as used memory; treat the reclaimable portion as cache, matching btop/htop.
+ memUsed = memInfo.Used - min(memInfo.Used, cgroup.ZFSARCReclaimable())
+ memTotal = memInfo.Total
+ }
+ cpuCount, memUsed, memTotal := h.applyCgroupLimits(ctx, h.systemStaticInfo.cpuCount, memUsed, memTotal)
+
+ diskUsagePath, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {
+ path := "/"
+ if h.systemService != nil {
+ path = h.systemService.GetDiskUsagePath(ctx)
+ }
+ return map[struct{}]string{{}: path}, nil
+ })
+ diskUsagePath = kit.Ternary(err != nil || !found, "/", diskUsagePath)
+ diskInfo, err := disk.Usage(diskUsagePath)
+ if (err != nil || diskInfo == nil || diskInfo.Total == 0) && diskUsagePath != "/" {
+ diskInfo, _ = disk.Usage("/")
+ }
+
+ var gpuStats []systemtypes.GPUStats
+ if h.gpuMonitor != nil && h.gpuMonitor.Enabled() {
+ if stats, statsErr := h.gpuMonitor.Stats(ctx); statsErr == nil {
+ gpuStats = stats
+ }
}
- // gopsutil counts ZFS ARC as used memory (the kernel excludes it from
- // MemAvailable). Treat the reclaimable portion as cache, matching
- // btop/htop, so the dashboard does not over-report usage on ZFS hosts.
- used := memInfo.Used
- if arc := cgroup.ZFSARCReclaimable(); arc > 0 {
- used -= min(used, arc)
+
+ return systemtypes.SystemStats{
+ CPUUsage: cpuUsage,
+ MemoryUsage: memUsed,
+ MemoryTotal: memTotal,
+ DiskUsage: kit.FromPtr(diskInfo).Used,
+ DiskTotal: kit.FromPtr(diskInfo).Total,
+ CPUCount: cpuCount,
+ Architecture: runtime.GOARCH,
+ Platform: runtime.GOOS,
+ Hostname: h.systemStaticInfo.hostname,
+ GPUCount: len(gpuStats),
+ GPUs: gpuStats,
}
- return used, memInfo.Total
}
// applyCgroupLimits uses guest resources without applying Docker's
// per-container limits to host statistics.
-func (h *WebSocketHandler) applyCgroupLimits(
- ctx context.Context, cpuCount int, memUsed, memTotal uint64,
-) (int, uint64, uint64) {
+func (h *WebSocketHandler) applyCgroupLimits(ctx context.Context, cpuCount int, memUsed, memTotal uint64) (int, uint64, uint64) {
if cgroup.IsDockerContainer() {
if h.systemService != nil {
if used, total, ok := h.systemService.GetDockerHostMemory(ctx); ok {
@@ -293,7 +258,7 @@ func (h *WebSocketHandler) applyCgroupLimits(
return cpuCount, memUsed, memTotal
}
cpuCount = applyCPUAffinityLimit(cpuCount)
- cgroupLimits := h.getCachedCgroupLimits()
+ cgroupLimits := h.cgroupCache.Get()
if cgroupLimits == nil {
return cpuCount, memUsed, memTotal
}
@@ -327,39 +292,6 @@ func applyCPUAffinityLimit(cpuCount int) int {
return cpuCount
}
-// getDiskInfo returns disk usage and total.
-func (h *WebSocketHandler) getDiskInfo(ctx context.Context) (uint64, uint64) {
- diskUsagePath := h.getDiskUsagePath(ctx)
- diskInfo, err := disk.Usage(diskUsagePath)
- if err != nil || diskInfo == nil || diskInfo.Total == 0 {
- if diskUsagePath != "/" {
- diskInfo, _ = disk.Usage("/")
- }
- }
- if diskInfo == nil {
- return 0, 0
- }
- return diskInfo.Used, diskInfo.Total
-}
-
-// getHostname returns the system hostname.
-func (h *WebSocketHandler) getHostname() string {
- h.initSystemStaticInfo()
- return h.systemStaticInfo.hostname
-}
-
-// getGPUInfo returns GPU statistics if monitoring is enabled.
-func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStats, int) {
- if h.gpuMonitor == nil || !h.gpuMonitor.Enabled() {
- return nil, 0
- }
- gpuData, err := h.gpuMonitor.Stats(ctx)
- if err != nil {
- return nil, 0
- }
- return gpuData, len(gpuData)
-}
-
// initializeCPUCache permits unavailable CPU samples but stops on cancellation.
func (h *WebSocketHandler) initializeCPUCache(ctx context.Context) bool {
result := make(chan float64, 1)
@@ -385,35 +317,16 @@ func (h *WebSocketHandler) initializeCPUCache(ctx context.Context) bool {
}
}
-func (h *WebSocketHandler) updateCPUCache(interval time.Duration) {
- if val, ok := h.readCPUUsage(interval); ok {
- h.cpuCache.Store(val)
- }
-}
-
func (h *WebSocketHandler) readCPUUsage(interval time.Duration) (float64, bool) {
if h.cpuUsageReader != nil {
return h.cpuUsageReader(interval)
}
-
- return defaultReadCPUUsage(interval)
-}
-
-func defaultReadCPUUsage(interval time.Duration) (float64, bool) {
if vals, err := cpu.Percent(interval, false); err == nil && len(vals) > 0 {
return vals[0], true
}
-
return 0, false
}
-func (h *WebSocketHandler) getCachedCgroupLimits() *cgroup.Limits {
- if h.cgroupCache == nil {
- return nil
- }
- return h.cgroupCache.Get()
-}
-
// SystemStats streams system stats over WebSocket.
//
// @Summary Get system stats via WebSocket
@@ -424,10 +337,25 @@ func (h *WebSocketHandler) getCachedCgroupLimits() *cgroup.Limits {
func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
clientIP := c.RealIP()
- if !h.checkRateLimit(clientIP) {
+ h.activeConnectionsMu.Lock()
+ if h.activeConnections[clientIP] >= 5 {
+ h.activeConnectionsMu.Unlock()
return c.JSON(http.StatusTooManyRequests, map[string]any{"success": false, "error": "Too many concurrent stats connections from this IP"})
}
- defer h.releaseRateLimit(clientIP)
+ if h.activeConnections == nil {
+ h.activeConnections = make(map[string]int)
+ }
+ h.activeConnections[clientIP]++
+ h.activeConnectionsMu.Unlock()
+ defer func() {
+ h.activeConnectionsMu.Lock()
+ defer h.activeConnectionsMu.Unlock()
+ if count := h.activeConnections[clientIP]; count > 1 {
+ h.activeConnections[clientIP] = count - 1
+ } else {
+ delete(h.activeConnections, clientIP)
+ }
+ }()
conn, unregister, ok := h.acceptWS(c, systemtypes.WSKindSystemStats, "")
if !ok {
@@ -459,8 +387,15 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
}
defer h.releaseSystemStatsSampler(sampleInterval)
- go h.readSystemStatsPump(ctx, cancel, conn)
- // The pong is serviced by readSystemStatsPump.
+ // Single reader for this connection; it also services the keepalive pong. Do not add others.
+ go func() {
+ for {
+ if _, _, err := conn.Read(ctx); err != nil {
+ cancel()
+ return
+ }
+ }
+ }()
go wshub.KeepAlive(ctx, cancel, conn)
send := func() error {
@@ -489,30 +424,3 @@ func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
}
}
}
-
-// readSystemStatsPump is the single reader for the SystemStats websocket.
-// Do not add additional readers for this connection.
-func (h *WebSocketHandler) readSystemStatsPump(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn) {
- for {
- if _, _, err := conn.Read(ctx); err != nil {
- cancel()
- return
- }
- }
-}
-
-func (h *WebSocketHandler) getDiskUsagePath(ctx context.Context) string {
- if h.diskUsagePathCache == nil {
- h.diskUsagePathCache = hot.NewHotCache[struct{}, string](hot.LRU, 1).
- WithTTL(5 * time.Minute).
- Build()
- }
- path, found, err := h.diskUsagePathCache.GetWithLoaders(struct{}{}, func(_ []struct{}) (map[struct{}]string, error) {
- path := "/"
- if h.systemService != nil {
- path = h.systemService.GetDiskUsagePath(ctx)
- }
- return map[struct{}]string{{}: path}, nil
- })
- return kit.Ternary(err != nil || !found, "/", path)
-}
diff --git a/backend/api/ws/system_stats_test.go b/backend/api/ws/system_stats_test.go
index a30e385b5..5190e9351 100644
--- a/backend/api/ws/system_stats_test.go
+++ b/backend/api/ws/system_stats_test.go
@@ -69,6 +69,7 @@ func TestSystemStatsSamplerEffectiveInterval(t *testing.T) {
func TestSystemStatsSamplerRearmsOnFasterSubscriber(t *testing.T) {
collections := make(chan struct{}, 16)
h := &WebSocketHandler{
+ ctx: t.Context(),
cpuUsageReader: func(time.Duration) (float64, bool) { return 0, true },
systemStatsCollector: func(context.Context) systemtypes.SystemStats {
select {
diff --git a/backend/internal/activity/children/pubsub/service.go b/backend/internal/activity/children/pubsub/service.go
new file mode 100644
index 000000000..822eb570f
--- /dev/null
+++ b/backend/internal/activity/children/pubsub/service.go
@@ -0,0 +1,287 @@
+// Package pubsub fans activity stream events out to per-environment subscribers.
+package pubsub
+
+import (
+ "cmp"
+ "context"
+ "log/slog"
+ "slices"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/samber/mo"
+)
+
+const (
+ // terminalPublishRetention bounds how long a terminal publish suppresses
+ // stale non-terminal snapshots for the same activity. Stale publishers are
+ // goroutines already past their commit, so they publish within moments; the
+ // retention only has to outlive that gap while keeping the latch map small.
+ terminalPublishRetention = 10 * time.Minute
+
+ // subscriberMessageQueueLimit bounds the per-subscriber backlog of "message"
+ // events; the oldest message is dropped (and flagged as missed) on overflow.
+ subscriberMessageQueueLimit = 256
+)
+
+// Service fans activity stream events out to per-environment subscribers.
+type Service struct {
+ subscribersMu sync.RWMutex
+ subscribers map[int]*subscriber
+ nextSubID int
+
+ // terminalPublished records when each activity's terminal snapshot was
+ // published, so later non-terminal snapshots from slower goroutines can
+ // be dropped (see admit). Entries are pruned on terminal publishes once
+ // older than terminalPublishRetention.
+ terminalPublishedMu sync.Mutex
+ terminalPublished map[string]time.Time
+}
+
+// subscriber buffers stream events between publishers and one stream
+// consumer. "activity" events are coalesced in place per activity ID (only the
+// latest pending state matters to the UI), so bulk operations emitting rapid
+// progress updates cannot overflow the subscriber and force full-snapshot
+// resends. Other events keep arrival order in a FIFO bounded by
+// subscriberMessageQueueLimit with drop-oldest on overflow.
+type subscriber struct {
+ environmentID string
+ ch chan activity.StreamEvent
+ done chan struct{}
+ wake chan struct{}
+
+ mu sync.Mutex
+ missed bool
+ queue []*pendingStreamEvent
+ pendingActivity map[string]*pendingStreamEvent
+ messageCount int
+}
+
+type pendingStreamEvent struct {
+ event activity.StreamEvent
+}
+
+// NewService returns an empty subscriber registry.
+func NewService() *Service {
+ return &Service{
+ subscribers: map[int]*subscriber{},
+ terminalPublished: map[string]time.Time{},
+ }
+}
+
+func isCoalescable(event activity.StreamEvent) bool {
+ return event.Type == "activity" && event.ActivityID != ""
+}
+
+func (sub *subscriber) enqueue(event activity.StreamEvent) {
+ sub.mu.Lock()
+ if isCoalescable(event) {
+ if pending, ok := sub.pendingActivity[event.ActivityID]; ok {
+ pending.event = event
+ sub.mu.Unlock()
+ return
+ }
+ } else {
+ if sub.messageCount >= subscriberMessageQueueLimit {
+ sub.dropOldestMessageLocked()
+ }
+ sub.messageCount++
+ }
+ entry := &pendingStreamEvent{event: event}
+ sub.queue = append(sub.queue, entry)
+ if isCoalescable(event) {
+ sub.pendingActivity[event.ActivityID] = entry
+ }
+ sub.mu.Unlock()
+
+ select {
+ case sub.wake <- struct{}{}:
+ default:
+ }
+}
+
+func (sub *subscriber) dropOldestMessageLocked() {
+ for i, entry := range sub.queue {
+ if isCoalescable(entry.event) {
+ continue
+ }
+ sub.queue = append(sub.queue[:i], sub.queue[i+1:]...)
+ sub.messageCount--
+ sub.missed = true
+ ctx := context.Background() //nolint:forbidigo // Subscriber buffer trimming runs outside any request context.
+ slog.WarnContext(ctx, "activity subscriber message buffer full; snapshot will be sent on next heartbeat",
+ "environmentId", sub.environmentID)
+ return
+ }
+}
+
+func (sub *subscriber) next() mo.Option[activity.StreamEvent] {
+ sub.mu.Lock()
+ defer sub.mu.Unlock()
+
+ if len(sub.queue) == 0 {
+ return mo.None[activity.StreamEvent]()
+ }
+ entry := sub.queue[0]
+ sub.queue = sub.queue[1:]
+ event := entry.event
+ if isCoalescable(event) {
+ if sub.pendingActivity[event.ActivityID] == entry {
+ delete(sub.pendingActivity, event.ActivityID)
+ }
+ } else {
+ sub.messageCount--
+ }
+ return mo.Some(event)
+}
+
+func (sub *subscriber) pump() {
+ defer close(sub.ch)
+ for {
+ event, ok := sub.next().Get()
+ if !ok {
+ select {
+ case <-sub.wake:
+ continue
+ case <-sub.done:
+ return
+ }
+ }
+ select {
+ case sub.ch <- event:
+ case <-sub.done:
+ return
+ }
+ }
+}
+
+// Subscribe returns the event channel for environmentID, a drop-check, and an unsubscribe func.
+func (s *Service) Subscribe(environmentID string) (<-chan activity.StreamEvent, func() bool, func()) {
+ ch := make(chan activity.StreamEvent, 64)
+ sub := &subscriber{
+ environmentID: cmp.Or(strings.TrimSpace(environmentID), "0"),
+ ch: ch,
+ done: make(chan struct{}),
+ wake: make(chan struct{}, 1),
+ pendingActivity: map[string]*pendingStreamEvent{},
+ }
+ s.subscribersMu.Lock()
+ s.nextSubID++
+ id := s.nextSubID
+ s.subscribers[id] = sub
+ s.subscribersMu.Unlock()
+ go sub.pump()
+
+ missedEvents := func() bool {
+ s.subscribersMu.RLock()
+ localSub, ok := s.subscribers[id]
+ s.subscribersMu.RUnlock()
+ if !ok {
+ return false
+ }
+
+ localSub.mu.Lock()
+ defer localSub.mu.Unlock()
+ if !localSub.missed {
+ return false
+ }
+ localSub.missed = false
+ return true
+ }
+
+ unsubscribe := func() {
+ s.subscribersMu.Lock()
+ localSub, ok := s.subscribers[id]
+ if ok {
+ delete(s.subscribers, id)
+ }
+ s.subscribersMu.Unlock()
+ if ok {
+ // The pump goroutine owns ch and closes it on shutdown.
+ close(localSub.done)
+ }
+ }
+
+ return ch, missedEvents, unsubscribe
+}
+
+// PublishActivity streams an activity snapshot unless a terminal snapshot for
+// the same activity was already published.
+func (s *Service) PublishActivity(item activity.Activity) {
+ if !s.admit(item) {
+ return
+ }
+ s.publish(item.EnvironmentID, activity.StreamEvent{
+ Type: "activity",
+ ActivityID: item.ID,
+ Activity: &item,
+ Timestamp: time.Now(),
+ })
+}
+
+// PublishMessage delivers one activity message to every subscriber of environmentID.
+func (s *Service) PublishMessage(environmentID string, activityType activity.Type, message activity.Message) {
+ s.publish(environmentID, activity.StreamEvent{
+ Type: "message",
+ ActivityID: message.ActivityID,
+ ActivityType: activityType,
+ Message: &message,
+ Timestamp: time.Now(),
+ })
+}
+
+// Reopen clears the terminal latch for an activity whose lifecycle can
+// legitimately restart, such as a retried job summary.
+func (s *Service) Reopen(activityID string) {
+ s.terminalPublishedMu.Lock()
+ delete(s.terminalPublished, activityID)
+ s.terminalPublishedMu.Unlock()
+}
+
+// admit orders activity events across the commit → publish gap. Writes
+// publish their snapshot after committing, so a goroutine that committed
+// before a terminal write can publish after it; since a terminal activity
+// emits no further events to correct the stream (and subscriber coalescing
+// would even replace an undelivered terminal event in place), such a stale
+// non-terminal snapshot must be dropped, not delivered. Activities never leave
+// a terminal status, so once a terminal snapshot is published, any later
+// non-terminal snapshot for that ID is stale by construction.
+func (s *Service) admit(item activity.Activity) bool {
+ s.terminalPublishedMu.Lock()
+ defer s.terminalPublishedMu.Unlock()
+ if !slices.Contains([]activity.Status{activity.StatusSuccess, activity.StatusFailed, activity.StatusCancelled}, item.Status) {
+ _, sealed := s.terminalPublished[item.ID]
+ return !sealed
+ }
+ now := time.Now()
+ for id, publishedAt := range s.terminalPublished {
+ if now.Sub(publishedAt) > terminalPublishRetention {
+ delete(s.terminalPublished, id)
+ }
+ }
+ s.terminalPublished[item.ID] = now
+ return true
+}
+
+func (s *Service) publish(environmentID string, event activity.StreamEvent) {
+ s.subscribersMu.RLock()
+ subs := make([]*subscriber, 0, len(s.subscribers))
+ for _, sub := range s.subscribers {
+ if sub.environmentID == environmentID {
+ subs = append(subs, sub)
+ }
+ }
+ s.subscribersMu.RUnlock()
+
+ for _, sub := range subs {
+ // Source labeling mutates activity fields independently for each subscriber.
+ subscriberEvent := event
+ if event.Activity != nil {
+ subscriberEvent.Activity = new(*event.Activity)
+ }
+ subscriberEvent.Activities = slices.Clone(event.Activities)
+ sub.enqueue(subscriberEvent)
+ }
+}
diff --git a/backend/internal/activity/children/pubsub/service_test.go b/backend/internal/activity/children/pubsub/service_test.go
new file mode 100644
index 000000000..d867f3616
--- /dev/null
+++ b/backend/internal/activity/children/pubsub/service_test.go
@@ -0,0 +1,102 @@
+package pubsub
+
+import (
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/activity"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSubscribeMarksMissedEventsWhenBufferFull(t *testing.T) {
+ service := NewService()
+
+ events, missedEvents, unsubscribe := service.Subscribe("0")
+ defer unsubscribe()
+
+ // Message-type events are not coalesced; overflowing the delivery channel
+ // plus the bounded pending FIFO must flag missed events so the stream
+ // handler resends a snapshot.
+ total := cap(events) + subscriberMessageQueueLimit + 50
+ for range total {
+ service.publish("0", activity.StreamEvent{Type: "message"})
+ }
+ require.True(t, missedEvents())
+ require.False(t, missedEvents())
+}
+
+func TestSubscriberCoalescesProgressEvents(t *testing.T) {
+ service := NewService()
+
+ events, missedEvents, unsubscribe := service.Subscribe("0")
+ defer unsubscribe()
+
+ const updates = 500
+ for i := 1; i <= updates; i++ {
+ progress := i * 100 / updates
+ service.PublishActivity(activity.Activity{
+ ID: "act-1",
+ EnvironmentID: "0",
+ Status: activity.StatusRunning,
+ Progress: &progress,
+ })
+ }
+
+ received := 0
+ deadline := time.After(5 * time.Second)
+ for {
+ select {
+ case event := <-events:
+ received++
+ require.Equal(t, "activity", event.Type)
+ if event.Activity != nil && event.Activity.Progress != nil && *event.Activity.Progress == 100 {
+ // The consumer was idle during publishing, so the backlog must
+ // have been coalesced instead of delivered event-for-event.
+ require.Less(t, received, updates)
+ require.False(t, missedEvents())
+ return
+ }
+ case <-deadline:
+ require.FailNow(t, "did not receive final coalesced progress event")
+ }
+ }
+}
+
+// A stale non-terminal snapshot published after the activity's terminal event
+// — a goroutine that committed before the terminal write but publishes after
+// it — is dropped instead of reverting subscribers to a running state no later
+// event would correct.
+func TestDropsStaleSnapshotAfterTerminalPublish(t *testing.T) {
+ service := NewService()
+
+ stale := activity.Activity{
+ ID: "act-1",
+ EnvironmentID: "0",
+ Type: activity.TypeImagePull,
+ Status: activity.StatusRunning,
+ }
+ completed := stale
+ completed.Status = activity.StatusSuccess
+
+ events, _, unsubscribe := service.Subscribe("0")
+ defer unsubscribe()
+
+ service.PublishActivity(completed)
+ select {
+ case event := <-events:
+ require.Equal(t, "activity", event.Type)
+ require.Equal(t, activity.StatusSuccess, event.Activity.Status)
+ case <-time.After(5 * time.Second):
+ require.FailNow(t, "terminal snapshot was not published")
+ }
+
+ require.False(t, service.admit(stale))
+ require.True(t, service.admit(completed))
+
+ service.PublishActivity(stale)
+ select {
+ case localEvent := <-events:
+ t.Fatalf("stale non-terminal snapshot reached subscriber: %+v", localEvent)
+ case <-time.After(100 * time.Millisecond):
+ }
+}
diff --git a/backend/internal/activity/children/slots/service.go b/backend/internal/activity/children/slots/service.go
new file mode 100644
index 000000000..691225cf0
--- /dev/null
+++ b/backend/internal/activity/children/slots/service.go
@@ -0,0 +1,161 @@
+// Package slots bounds how many queue-opted activities run concurrently per
+// environment and tracks which activity holds each slot.
+package slots
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/samber/mo"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
+)
+
+const (
+ maxConcurrentActivitiesSettingKey = "maxConcurrentActivities"
+ defaultMaxConcurrentActivities = 5
+ // slotWaitRecheckInterval bounds how long a queued waiter can miss a limit
+ // increase made while no slots were being released.
+ slotWaitRecheckInterval = 2 * time.Second
+)
+
+// Service reads the limit from settings on every acquisition attempt, while
+// held slots are counted independently of it — so changing the limit
+// mid-flight never miscounts running work: existing holders keep occupying
+// capacity until they release, and waiters are re-evaluated against the new
+// limit.
+type Service struct {
+ settings *settings.SettingsService
+
+ mu sync.Mutex
+ state map[string]*environmentSlots
+
+ // releases maps an activity ID to the release func of the slot it holds;
+ // the slot is freed when the activity completes.
+ releasesMu sync.Mutex
+ releases map[string]func()
+}
+
+type environmentSlots struct {
+ held int
+ // wake is closed (and replaced) on every release to broadcast to waiters;
+ // each re-checks against the current limit, so a raised limit can admit
+ // more than one of them.
+ wake chan struct{}
+}
+
+// NewService returns a slot limiter sized from the local settings.
+func NewService(localSettings *settings.SettingsService) *Service {
+ return &Service{
+ settings: localSettings,
+ state: map[string]*environmentSlots{},
+ releases: map[string]func(){},
+ }
+}
+
+func (s *Service) stateLocked(environmentID string) *environmentSlots {
+ slots := s.state[environmentID]
+ if slots == nil {
+ slots = &environmentSlots{wake: make(chan struct{})}
+ s.state[environmentID] = slots
+ }
+ return slots
+}
+
+// TryAcquire grabs a slot without blocking; None means the caller should
+// queue and block via Await. Held slots are counted even while the limit is
+// unlimited, so enabling a limit later still sees the work already running.
+func (s *Service) TryAcquire(ctx context.Context, environmentID string) mo.Option[func()] {
+ if s.settings == nil {
+ return mo.Some(func() {})
+ }
+ limit := s.settings.GetIntSetting(ctx, maxConcurrentActivitiesSettingKey, defaultMaxConcurrentActivities)
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ slots := s.stateLocked(environmentID)
+ if limit > 0 && slots.held >= limit {
+ return mo.None[func()]()
+ }
+ slots.held++
+ return mo.Some(s.releaser(environmentID))
+}
+
+// Await blocks until activityID holds a slot or ctx is cancelled, reporting
+// whether a slot was newly acquired. The limit is re-read on every attempt so
+// setting changes apply to queued waiters.
+func (s *Service) Await(ctx context.Context, activityID, environmentID string) (bool, error) {
+ s.releasesMu.Lock()
+ _, held := s.releases[activityID]
+ s.releasesMu.Unlock()
+ if held {
+ return false, nil
+ }
+
+ for s.settings != nil {
+ limit := s.settings.GetIntSetting(ctx, maxConcurrentActivitiesSettingKey, defaultMaxConcurrentActivities)
+ s.mu.Lock()
+ slots := s.stateLocked(environmentID)
+ if limit <= 0 || slots.held < limit {
+ slots.held++
+ s.mu.Unlock()
+ s.Register(activityID, s.releaser(environmentID))
+ return true, nil
+ }
+ wake := slots.wake
+ s.mu.Unlock()
+
+ timer := time.NewTimer(slotWaitRecheckInterval)
+ select {
+ case <-wake:
+ case <-timer.C:
+ case <-ctx.Done():
+ timer.Stop()
+ return false, context.Cause(ctx)
+ }
+ timer.Stop()
+ }
+ s.Register(activityID, func() {})
+ return true, nil
+}
+
+// Register records release as the slot held by activityID, freeing any slot
+// it previously held.
+func (s *Service) Register(activityID string, release func()) {
+ s.releasesMu.Lock()
+ if existing, ok := s.releases[activityID]; ok {
+ existing()
+ }
+ s.releases[activityID] = release
+ s.releasesMu.Unlock()
+}
+
+// Release frees the slot held by activityID, if any.
+func (s *Service) Release(activityID string) {
+ s.releasesMu.Lock()
+ release, ok := s.releases[activityID]
+ if ok {
+ delete(s.releases, activityID)
+ }
+ s.releasesMu.Unlock()
+ if ok {
+ release()
+ }
+}
+
+func (s *Service) releaser(environmentID string) func() {
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ s.mu.Lock()
+ slots := s.stateLocked(environmentID)
+ if slots.held > 0 {
+ slots.held--
+ }
+ close(slots.wake)
+ slots.wake = make(chan struct{})
+ s.mu.Unlock()
+ })
+ }
+}
diff --git a/backend/internal/activity/children/sweeps/service.go b/backend/internal/activity/children/sweeps/service.go
new file mode 100644
index 000000000..873c7f831
--- /dev/null
+++ b/backend/internal/activity/children/sweeps/service.go
@@ -0,0 +1,115 @@
+// Package sweeps selects the activity rows that periodic and startup repair
+// sweeps finalize.
+package sweeps
+
+import (
+ "context"
+ "slices"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/activity"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+)
+
+const (
+ staleImageUpdateCheckAge = 6 * time.Hour
+ // abandonedActivityGrace is how old an untracked queued/running activity
+ // must be before the periodic sweep fails it. It covers the window between
+ // StartActivity's row creation and the worker's Track call.
+ abandonedActivityGrace = 2 * time.Minute
+
+ // abandonedActivitySweepLimit bounds one sweep; later sweeps take the rest.
+ abandonedActivitySweepLimit = 500
+)
+
+var backupActions = []string{
+ "create_volume_backup",
+ "create_system_backup",
+ "run_system_volume_backups",
+ "scheduled_volume_backup",
+ "scheduled_system_backup",
+}
+
+// Candidate is the part of an activity row a sweep decides on.
+type Candidate struct {
+ ID string
+ StartedAt time.Time
+ Metadata database.JSON `gorm:"type:text"`
+}
+
+// Service runs the candidate queries behind the periodic activity sweeps.
+type Service struct {
+ db *database.DB
+}
+
+// NewService returns a sweeps query service over db.
+func NewService(db *database.DB) *Service {
+ return &Service{db: db}
+}
+
+func (s *Service) activities(ctx context.Context) *gorm.DB {
+ return s.db.WithContext(ctx).Table("activities")
+}
+
+// StaleImageUpdateChecks returns image update checks left running across a
+// prior process lifetime.
+func (s *Service) StaleImageUpdateChecks(ctx context.Context) ([]Candidate, error) {
+ var rows []Candidate
+ err := s.activities(ctx).
+ Where("type = ? AND status = ? AND started_at < ?", activity.TypeImageUpdateCheck, activity.StatusRunning, time.Now().Add(-staleImageUpdateCheckAge)).
+ Find(&rows).Error
+ return rows, err
+}
+
+// Abandoned returns active non-job activities past the grace period that are
+// not in trackedIDs, oldest first.
+func (s *Service) Abandoned(ctx context.Context, trackedIDs []string) ([]Candidate, error) {
+ query := s.activities(ctx).
+ Select("id", "started_at").
+ Where("status IN ? AND started_at < ?", []activity.Status{activity.StatusQueued, activity.StatusRunning}, time.Now().Add(-abandonedActivityGrace)).
+ Where("type <> ?", activity.TypeJobRun)
+ if len(trackedIDs) > 0 {
+ query = query.Where("id NOT IN ?", trackedIDs)
+ }
+ var rows []Candidate
+ err := query.Order("started_at").Limit(abandonedActivitySweepLimit).Find(&rows).Error
+ return rows, err
+}
+
+// OrphanedQueued returns queued non-job activities outside protectedIDs.
+func (s *Service) OrphanedQueued(ctx context.Context, protectedIDs []string) ([]Candidate, error) {
+ var rows []Candidate
+ err := s.activities(ctx).
+ Where("status = ?", activity.StatusQueued).
+ Where("type <> ?", activity.TypeJobRun).
+ Where("id NOT IN ?", append(protectedIDs, "")).
+ Find(&rows).Error
+ return rows, err
+}
+
+// StaleAutoUpdates returns running auto-update activities outside protectedIDs.
+func (s *Service) StaleAutoUpdates(ctx context.Context, protectedIDs []string) ([]Candidate, error) {
+ var rows []Candidate
+ err := s.activities(ctx).
+ Where("type = ? AND status = ?", activity.TypeAutoUpdate, activity.StatusRunning).
+ Where("id NOT IN ?", append(protectedIDs, "")).
+ Find(&rows).Error
+ return rows, err
+}
+
+// InterruptedBackups returns active backup activities outside protectedIDs.
+func (s *Service) InterruptedBackups(ctx context.Context, protectedIDs []string) ([]Candidate, error) {
+ var rows []Candidate
+ if err := s.activities(ctx).
+ Where("status IN ?", []activity.Status{activity.StatusQueued, activity.StatusRunning}).
+ Where("id NOT IN ?", append(protectedIDs, "")).
+ Find(&rows).Error; err != nil {
+ return nil, err
+ }
+ return slices.DeleteFunc(rows, func(row Candidate) bool {
+ action, _ := row.Metadata["action"].(string)
+ return !slices.Contains(backupActions, action)
+ }), nil
+}
diff --git a/backend/internal/activity/handler.go b/backend/internal/activity/handler.go
index 6371f05d3..5704051a2 100644
--- a/backend/internal/activity/handler.go
+++ b/backend/internal/activity/handler.go
@@ -91,16 +91,38 @@ func NewHandler(activityService *ActivityService, localEnvironment EnvironmentDe
func (h *ActivityHandler) ListActivities(ctx context.Context, input *ListActivitiesInput) (*handlerutil.Page[activity.Activity], error) {
if input.EnvironmentID != "0" {
- return h.proxyListActivitiesInternal(ctx, input)
+ unavailable := false
+ out, err := h.listRemoteActivities(ctx, input, func(path string) (*base.Paginated[activity.Activity], error) {
+ var remote base.Paginated[activity.Activity]
+ if proxyErr := h.environment.ProxyJSONRequest(ctx, input.EnvironmentID, http.MethodGet, path, nil, &remote); proxyErr != nil {
+ var transportErr *remenv.TransportError
+ unavailable = errors.As(proxyErr, &transportErr) && transportErr.Unavailable()
+ return nil, handlerutil.TranslateRemoteProxyError(proxyErr)
+ }
+ return &remote, nil
+ })
+ if out == nil {
+ return nil, err
+ }
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return nil, ctxErr
+ }
+ if err != nil && !unavailable {
+ return nil, err
+ }
+ // HTTP responses must not discard stored summaries when the agent is offline.
+ // Stream pollers retain the proxy error from the shared listing path.
+ out.Body.Success = true
+ return out, nil
}
- params := activityListParamsInternal(input)
+ params := activityListParams(input)
activities, paginationResp, err := h.activityService.ListActivitiesPaginated(ctx, input.EnvironmentID, params)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)
+ h.applyActivitySourceLabels(ctx, input.EnvironmentID, activities)
return &handlerutil.Page[activity.Activity]{
Body: base.Paginated[activity.Activity]{
@@ -120,16 +142,22 @@ func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInp
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
if input.EnvironmentID != "0" {
- return h.proxyGetActivityInternal(ctx, input)
+ path := fmt.Sprintf("/api/environments/0/activities/%s?limit=%d", url.PathEscape(input.ActivityID), input.Limit)
+ out, proxyErr := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Detail]](ctx, input.EnvironmentID, http.MethodGet, path, nil)
+ if proxyErr != nil {
+ return nil, proxyErr
+ }
+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, &out.Data.Activity)
+ return &handlerutil.Out[activity.Detail]{Body: *out}, nil
}
return nil, huma.Error404NotFound("activity not found")
}
return nil, huma.Error500InternalServerError(err.Error())
}
- if !canReadJobActivityInternal(ctx, detail.Activity) {
+ if !canReadJobActivity(ctx, detail.Activity) {
return nil, huma.Error404NotFound("activity not found")
}
- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &detail.Activity)
+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, &detail.Activity)
return &handlerutil.Out[activity.Detail]{
Body: base.ApiResponse[activity.Detail]{
@@ -146,12 +174,12 @@ func (h *ActivityHandler) ClearHistory(ctx context.Context, input *ClearActivity
}
if input.EnvironmentID != "0" {
- out, proxyClearHistoryErr := h.proxyClearHistoryInternal(ctx, input)
- if proxyClearHistoryErr != nil {
- return nil, proxyClearHistoryErr
+ out, proxyErr := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.ClearHistoryResult]](ctx, input.EnvironmentID, http.MethodDelete, "/api/environments/0/activities/history", nil)
+ if proxyErr != nil {
+ return nil, proxyErr
}
- out.Body.Data.Deleted += deleted
- return out, nil
+ out.Data.Deleted += deleted
+ return &handlerutil.Out[activity.ClearHistoryResult]{Body: *out}, nil
}
return &handlerutil.Out[activity.ClearHistoryResult]{
@@ -167,13 +195,26 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActiv
return nil, huma.Error400BadRequest("activity id is required")
}
- requestedBy := h.cancelRequestedByInternal(ctx, input.RequestedBy)
+ // Prefer the authenticated user; a proxying manager forwards its user's name.
+ requestedBy := strings.TrimSpace(input.RequestedBy)
+ if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {
+ requestedBy = cmp.Or(strings.TrimSpace(kit.FromPtr(user.DisplayName)), strings.TrimSpace(user.Username), requestedBy)
+ }
cancelled, err := h.activityService.CancelActivity(ctx, input.EnvironmentID, input.ActivityID, requestedBy)
if err != nil {
switch {
case errors.Is(err, gorm.ErrRecordNotFound):
if input.EnvironmentID != "0" {
- return h.proxyCancelActivityInternal(ctx, input)
+ path := fmt.Sprintf("/api/environments/0/activities/%s/cancel", url.PathEscape(input.ActivityID))
+ if requestedBy != "" {
+ path += "?requestedBy=" + url.QueryEscape(requestedBy)
+ }
+ out, proxyErr := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Activity]](ctx, input.EnvironmentID, http.MethodPost, path, nil)
+ if proxyErr != nil {
+ return nil, proxyErr
+ }
+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, &out.Data)
+ return &handlerutil.Out[activity.Activity]{Body: *out}, nil
}
return nil, huma.Error404NotFound("activity not found")
case errors.Is(err, ErrActivityNotCancelable):
@@ -182,7 +223,7 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActiv
return nil, huma.Error500InternalServerError(err.Error())
}
}
- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, cancelled)
+ h.applyActivitySourceLabel(ctx, input.EnvironmentID, cancelled)
return &handlerutil.Out[activity.Activity]{
Body: base.ApiResponse[activity.Activity]{
@@ -192,38 +233,10 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActiv
}, nil
}
-func (h *ActivityHandler) proxyCancelActivityInternal(ctx context.Context, input *CancelActivityInput) (*handlerutil.Out[activity.Activity], error) {
- path := fmt.Sprintf("/api/environments/0/activities/%s/cancel", url.PathEscape(input.ActivityID))
- if requestedBy := h.cancelRequestedByInternal(ctx, input.RequestedBy); requestedBy != "" {
- path += "?requestedBy=" + url.QueryEscape(requestedBy)
- }
- out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Activity]](ctx, input.EnvironmentID, http.MethodPost, path, nil)
- if err != nil {
- return nil, err
- }
- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &out.Data)
- return &handlerutil.Out[activity.Activity]{Body: *out}, nil
-}
-
-// cancelRequestedByInternal resolves a human-readable name for the cancellation
-// audit message, preferring the authenticated user and falling back to a name
-// forwarded from a proxying controller.
-func (h *ActivityHandler) cancelRequestedByInternal(ctx context.Context, forwarded string) string {
- if user, ok := userctx.CurrentUserFromContext(ctx); ok && user != nil {
- if user.DisplayName != nil && strings.TrimSpace(*user.DisplayName) != "" {
- return strings.TrimSpace(*user.DisplayName)
- }
- if name := strings.TrimSpace(user.Username); name != "" {
- return name
- }
- }
- return strings.TrimSpace(forwarded)
-}
-
func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int, events chan<- activity.StreamEvent) {
sendSnapshot := func() bool {
activities, _, err := h.activityService.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{
- Limit: resolveActivityStreamLimitInternal(limit),
+ Limit: resolveActivityStreamLimit(limit),
SkipCount: true,
})
if err != nil {
@@ -237,7 +250,7 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,
}
return false
}
- h.applyActivitySourceLabelsInternal(ctx, "0", activities)
+ h.applyActivitySourceLabels(ctx, "0", activities)
return agg.Send(ctx, events, activity.StreamEvent{
Type: "snapshot",
EnvironmentID: "0",
@@ -248,7 +261,7 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,
snapshotOK := sendSnapshot()
- eventCh, missedEvents, unsubscribe := h.activityService.Subscribe("0")
+ eventCh, missedEvents, unsubscribe := h.activityService.pubsub.Subscribe("0")
defer unsubscribe()
ticker := time.NewTicker(activityStreamHeartbeatInterval)
@@ -262,11 +275,16 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,
if !ok {
return
}
- if !h.canReadActivityStreamEventInternal(ctx, event) {
+ if !h.canReadActivityStreamEvent(ctx, event) {
continue
}
event.EnvironmentID = "0"
- h.applyActivityStreamEventSourceLabelInternal(ctx, "0", &event)
+ if event.Activity != nil {
+ h.applyActivitySourceLabel(ctx, "0", event.Activity)
+ }
+ if len(event.Activities) > 0 {
+ h.applyActivitySourceLabels(ctx, "0", event.Activities)
+ }
if !agg.Send(ctx, events, event) {
return
}
@@ -299,17 +317,17 @@ func (h *ActivityHandler) RunRemoteStreamPollers(ctx context.Context, ps *authz.
func(environment environment.Environment) string {
return environment.ID
},
- activityStreamEnvironmentVersionInternal,
+ activityStreamEnvironmentVersion,
activityStreamEnvReconcileInterval,
"activity stream",
func(pollCtx context.Context, environment environment.Environment) {
// One shared poller per environment (and snapshot limit) serves
// every connected client; this subscriber only forwards its
// events onto this client's stream.
- key := activityStreamEnvironmentVersionInternal(environment) + ":" + strconv.Itoa(resolveActivityStreamLimitInternal(limit))
+ key := activityStreamEnvironmentVersion(environment) + ":" + strconv.Itoa(resolveActivityStreamLimit(limit))
h.remoteStreamHub.Subscribe(pollCtx, key,
func(runCtx context.Context, publish func(activity.StreamEvent)) {
- h.runRemoteActivityStreamPollerInternal(runCtx, environment, limit, publish)
+ h.runRemoteActivityStreamPoller(runCtx, environment, limit, publish)
},
func(event activity.StreamEvent) bool {
return agg.Send(pollCtx, events, event)
@@ -317,14 +335,14 @@ func (h *ActivityHandler) RunRemoteStreamPollers(ctx context.Context, ps *authz.
})
}
-func activityStreamEnvironmentVersionInternal(localEnvironment environment.Environment) string {
+func activityStreamEnvironmentVersion(localEnvironment environment.Environment) string {
if localEnvironment.UpdatedAt == nil {
return localEnvironment.ID
}
return localEnvironment.ID + ":" + localEnvironment.UpdatedAt.UTC().Format(time.RFC3339Nano)
}
-func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, localEnvironment environment.Environment, limit int, publish func(activity.StreamEvent)) {
+func (h *ActivityHandler) runRemoteActivityStreamPoller(ctx context.Context, localEnvironment environment.Environment, limit int, publish func(activity.StreamEvent)) {
environmentID := localEnvironment.ID
lastError := ""
lastFingerprint := ""
@@ -339,56 +357,58 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont
}
}
- output, err := h.proxyListActivitiesForEnvironmentInternal(ctx, currentEnvironment, &ListActivitiesInput{
+ output, err := h.listRemoteActivities(ctx, &ListActivitiesInput{
EnvironmentID: environmentID,
- Limit: resolveActivityStreamLimitInternal(limit),
+ Limit: resolveActivityStreamLimit(limit),
Order: "desc",
- })
- if err != nil {
- if ctx.Err() != nil {
- return
+ }, func(path string) (*base.Paginated[activity.Activity], error) {
+ pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)
+ defer cancelPoll()
+ var out base.Paginated[activity.Activity]
+ if err := h.environment.ProxyJSONRequestForEnvironment(pollCtx, currentEnvironment, http.MethodGet, path, nil, &out); err != nil {
+ return nil, handlerutil.TranslateRemoteProxyError(err)
}
- if output != nil {
- fingerprint := strconv.FormatUint(kit.Fingerprint(output.Body.Data), 16)
- if fingerprint != lastFingerprint {
- publish(activity.StreamEvent{
- Type: "snapshot",
- EnvironmentID: environmentID,
- Activities: output.Body.Data,
- Timestamp: time.Now(),
- })
- lastFingerprint = fingerprint
- lastError = ""
- }
+ return &out, nil
+ })
+ if err != nil && ctx.Err() != nil {
+ return
+ }
+ if err == nil && lastError != "" {
+ // Resend the snapshot once the environment recovers.
+ lastFingerprint = ""
+ }
+ if output != nil {
+ sourceID := cmp.Or(currentEnvironment.ID, environment.LocalEnvironmentID)
+ sourceName := environment.DisplayName(sourceID, currentEnvironment.Name)
+ for i := range output.Body.Data {
+ applyActivitySource(&output.Body.Data[i], sourceID, sourceName)
}
- // A failing environment must not end the stream; surface the error
- // once per distinct message and keep polling.
- if msg := err.Error(); msg != lastError {
- lastError = msg
+ if fingerprint := strconv.FormatUint(kit.Fingerprint(output.Body.Data), 16); fingerprint != lastFingerprint {
publish(activity.StreamEvent{
- Type: "error",
+ Type: "snapshot",
EnvironmentID: environmentID,
- Error: msg,
+ Activities: output.Body.Data,
Timestamp: time.Now(),
})
+ lastFingerprint = fingerprint
+ lastError = ""
}
- return
}
- if lastError != "" {
- lastFingerprint = ""
- }
- lastError = ""
- fingerprint := strconv.FormatUint(kit.Fingerprint(output.Body.Data), 16)
- if fingerprint == lastFingerprint {
+ if err == nil {
+ lastError = ""
return
}
- publish(activity.StreamEvent{
- Type: "snapshot",
- EnvironmentID: environmentID,
- Activities: output.Body.Data,
- Timestamp: time.Now(),
- })
- lastFingerprint = fingerprint
+ // A failing environment must not end the stream; surface the error
+ // once per distinct message and keep polling.
+ if msg := err.Error(); msg != lastError {
+ lastError = msg
+ publish(activity.StreamEvent{
+ Type: "error",
+ EnvironmentID: environmentID,
+ Error: msg,
+ Timestamp: time.Now(),
+ })
+ }
}
poll()
@@ -406,73 +426,24 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont
}
}
-func (h *ActivityHandler) proxyGetActivityInternal(ctx context.Context, input *GetActivityInput) (*handlerutil.Out[activity.Detail], error) {
- path := fmt.Sprintf("/api/environments/0/activities/%s?limit=%d", url.PathEscape(input.ActivityID), input.Limit)
- out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.Detail]](ctx, input.EnvironmentID, http.MethodGet, path, nil)
- if err != nil {
- return nil, err
- }
- h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &out.Data.Activity)
- return &handlerutil.Out[activity.Detail]{Body: *out}, nil
-}
-
-func (h *ActivityHandler) proxyClearHistoryInternal(ctx context.Context, input *ClearActivityHistoryInput) (*handlerutil.Out[activity.ClearHistoryResult], error) {
- out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activity.ClearHistoryResult]](ctx, input.EnvironmentID, http.MethodDelete, "/api/environments/0/activities/history", nil)
- if err != nil {
- return nil, err
- }
- return &handlerutil.Out[activity.ClearHistoryResult]{Body: *out}, nil
-}
-
-func (h *ActivityHandler) applyActivitySourceLabelsInternal(ctx context.Context, environmentID string, activities []activity.Activity) {
- sourceID, sourceName := h.resolveActivitySourceInternal(ctx, environmentID)
- for i := range activities {
- applyActivitySourceInternal(&activities[i], sourceID, sourceName)
- }
-}
-
-func (h *ActivityHandler) applyActivitySourceLabelInternal(ctx context.Context, environmentID string, item *activity.Activity) {
- sourceID, sourceName := h.resolveActivitySourceInternal(ctx, environmentID)
- applyActivitySourceInternal(item, sourceID, sourceName)
-}
-
-func (h *ActivityHandler) applyActivityStreamEventSourceLabelInternal(ctx context.Context, environmentID string, event *activity.StreamEvent) {
- if event == nil || (event.Activity == nil && len(event.Activities) == 0) {
- return
- }
- sourceID, sourceName := h.resolveActivitySourceInternal(ctx, environmentID)
- if event.Activity != nil {
- applyActivitySourceInternal(event.Activity, sourceID, sourceName)
- }
- for i := range event.Activities {
- applyActivitySourceInternal(&event.Activities[i], sourceID, sourceName)
+func (h *ActivityHandler) applyActivitySourceLabels(ctx context.Context, environmentID string, activities []activity.Activity) {
+ sourceID := cmp.Or(environmentID, environment.LocalEnvironmentID)
+ sourceName := environment.DisplayName(sourceID, "")
+ if h.environment.ResolveEnvironmentName != nil {
+ sourceName = h.environment.ResolveEnvironmentName(ctx, sourceID)
}
-}
-
-func applyActivitySourceLabelsForEnvironmentInternal(environmentModel environment.Environment, activities []activity.Activity) {
- sourceID, sourceName := activitySourceFromEnvironmentInternal(environmentModel)
for i := range activities {
- applyActivitySourceInternal(&activities[i], sourceID, sourceName)
+ applyActivitySource(&activities[i], sourceID, sourceName)
}
}
-func activitySourceFromEnvironmentInternal(environmentModel environment.Environment) (string, string) {
- environmentID := cmp.Or(environmentModel.ID, environment.LocalEnvironmentID)
- return environmentID, environment.DisplayName(environmentID, environmentModel.Name)
-}
-
-func (h *ActivityHandler) resolveActivitySourceInternal(ctx context.Context, environmentID string) (string, string) {
- environmentID = cmp.Or(environmentID, environment.LocalEnvironmentID)
- if h.environment.ResolveEnvironmentName != nil {
- return environmentID, h.environment.ResolveEnvironmentName(ctx, environmentID)
- }
- return environmentID, environment.DisplayName(environmentID, "")
+func (h *ActivityHandler) applyActivitySourceLabel(ctx context.Context, environmentID string, item *activity.Activity) {
+ items := []activity.Activity{*item}
+ h.applyActivitySourceLabels(ctx, environmentID, items)
+ *item = items[0]
}
-func applyActivitySourceInternal(item *activity.Activity, sourceID, sourceName string) {
- if item == nil {
- return
- }
+func applyActivitySource(item *activity.Activity, sourceID, sourceName string) {
item.SourceEnvironmentID = sourceID
item.SourceEnvironmentName = sourceName
item.EnvironmentID = sourceID
@@ -482,120 +453,48 @@ func applyActivitySourceInternal(item *activity.Activity, sourceID, sourceName s
}
}
-func activityListQueryInternal(input *ListActivitiesInput) url.Values {
- values := url.Values{}
- values.Set("start", strconv.Itoa(input.Start))
- values.Set("limit", strconv.Itoa(input.Limit))
- if input.Search != "" {
- values.Set("search", input.Search)
- }
- if input.Sort != "" {
- values.Set("sort", input.Sort)
- }
- if input.Order != "" {
- values.Set("order", input.Order)
- }
- if input.Status != "" {
- values.Set("status", input.Status)
- }
- if input.Type != "" {
- values.Set("type", input.Type)
- }
- if input.ResourceType != "" {
- values.Set("resourceType", input.ResourceType)
- }
- return values
-}
-
-func resolveActivityStreamLimitInternal(limit int) int {
- if limit <= 0 {
- return 50
- }
- return kit.Ternary(limit > 100, 100, limit)
+func resolveActivityStreamLimit(limit int) int {
+ return min(kit.Ternary(limit <= 0, 50, limit), 100)
}
-func activityListParamsInternal(input *ListActivitiesInput) pagination.QueryParams {
+// activityListParams maps list input onto pagination params and its filters.
+func activityListParams(input *ListActivitiesInput) pagination.QueryParams {
params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)
- params.Filters["status"] = input.Status
- params.Filters["type"] = input.Type
- params.Filters["resourceType"] = input.ResourceType
- return params
-}
-
-func (h *ActivityHandler) proxyListActivitiesInternal(ctx context.Context, input *ListActivitiesInput) (*handlerutil.Page[activity.Activity], error) {
- unavailable := false
- out, err := h.listRemoteActivitiesInternal(ctx, input, func(path string) (*base.Paginated[activity.Activity], error) {
- var remote base.Paginated[activity.Activity]
- if proxyErr := h.environment.ProxyJSONRequest(ctx, input.EnvironmentID, http.MethodGet, path, nil, &remote); proxyErr != nil {
- var transportErr *remenv.TransportError
- unavailable = errors.As(proxyErr, &transportErr) && transportErr.Unavailable()
- return nil, handlerutil.TranslateRemoteProxyError(proxyErr)
- }
- return &remote, nil
+ maps.Copy(params.Filters, map[string]string{
+ "status": input.Status,
+ "type": input.Type,
+ "resourceType": input.ResourceType,
})
- if out == nil {
- return nil, err
- }
- if ctxErr := ctx.Err(); ctxErr != nil {
- return nil, ctxErr
- }
- if err != nil && !unavailable {
- return nil, err
- }
- // HTTP responses must not discard stored summaries when the agent is offline.
- // Stream pollers retain the proxy error from the shared listing path.
- out.Body.Success = true
- return out, nil
-}
-
-func (
- h *ActivityHandler,
-) proxyListActivitiesForEnvironmentInternal(
- ctx context.Context,
- localEnvironment environment.Environment,
- input *ListActivitiesInput,
-) (
- *handlerutil.Page[activity.Activity],
- error,
-) {
- out, err := h.listRemoteActivitiesInternal(ctx, input, func(path string) (*base.Paginated[activity.Activity], error) {
- pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)
- defer cancelPoll()
- var out base.Paginated[activity.Activity]
- if err := h.environment.ProxyJSONRequestForEnvironment(pollCtx, localEnvironment, http.MethodGet, path, nil, &out); err != nil {
- return nil, handlerutil.TranslateRemoteProxyError(err)
- }
- return &out, nil
- })
- if out != nil {
- applyActivitySourceLabelsForEnvironmentInternal(localEnvironment, out.Body.Data)
- }
- return out, err
+ return params
}
-func (
- h *ActivityHandler,
-) listRemoteActivitiesInternal(
+func (h *ActivityHandler) listRemoteActivities(
ctx context.Context,
input *ListActivitiesInput,
- fetch func(
- string,
- ) (
- *base.Paginated[activity.Activity],
- error,
- ),
-) (
- *handlerutil.Page[activity.Activity],
- error,
-) {
- params := normalizeRemoteActivityParamsInternal(activityListParamsInternal(input))
+ fetch func(string) (*base.Paginated[activity.Activity], error),
+) (*handlerutil.Page[activity.Activity], error) {
+ params := normalizeRemoteActivityParams(activityListParams(input))
// The agent only ships the newest window the merged page can use, not its
// whole history; the stream poller repeats this every few seconds.
window := kit.Ternary(params.Limit == -1, -1, params.Start+params.Limit)
- remoteActivities, remoteTotal, remoteErr := collectActivityWindowInternal(window, func(start, limit int) ([]activity.Activity, int64, error) {
- page := *input
- page.Start, page.Limit = start, limit
- remote, err := fetch("/api/environments/0/activities?" + activityListQueryInternal(&page).Encode())
+ remoteActivities, remoteTotal, remoteErr := collectActivityWindow(window, func(start, limit int) ([]activity.Activity, int64, error) {
+ query := url.Values{
+ "start": {strconv.Itoa(start)},
+ "limit": {strconv.Itoa(limit)},
+ }
+ for key, value := range map[string]string{
+ "search": input.Search,
+ "sort": input.Sort,
+ "order": input.Order,
+ "status": input.Status,
+ "type": input.Type,
+ "resourceType": input.ResourceType,
+ } {
+ if value != "" {
+ query.Set(key, value)
+ }
+ }
+ remote, err := fetch("/api/environments/0/activities?" + query.Encode())
if err != nil {
return nil, 0, err
}
@@ -607,7 +506,7 @@ func (
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)
+ h.applyActivitySourceLabels(ctx, input.EnvironmentID, activities)
return &handlerutil.Page[activity.Activity]{Body: base.Paginated[activity.Activity]{
Success: remoteErr == nil,
Data: activities,
diff --git a/backend/internal/activity/helpers.go b/backend/internal/activity/helpers.go
index 1ff5f22fe..387c54530 100644
--- a/backend/internal/activity/helpers.go
+++ b/backend/internal/activity/helpers.go
@@ -3,136 +3,16 @@ package activity
import (
"cmp"
"context"
- "sync"
- "time"
"github.com/getarcaneapp/arcane/types/v2/activity"
- "github.com/samber/mo"
"gorm.io/gorm"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
)
-const (
- maxConcurrentActivitiesSettingKey = "maxConcurrentActivities"
- defaultMaxConcurrentActivities = 5
- // slotWaitRecheckInterval bounds how long a queued waiter can miss a limit
- // increase made while no slots were being released.
- slotWaitRecheckInterval = 2 * time.Second
-)
-
-// activitySlotLimiter bounds how many queue-opted activities run concurrently
-// per environment. The limit is read from settings on every acquisition
-// attempt, while held slots are counted independently of it — so changing the
-// limit mid-flight never miscounts running work: existing holders keep
-// occupying capacity until they release, and waiters are re-evaluated against
-// the new limit.
-type activitySlotLimiter struct {
- settings *settings.SettingsService
-
- mu sync.Mutex
- state map[string]*activityEnvironmentSlots
-}
-
-type activityEnvironmentSlots struct {
- held int
- // wake is closed (and replaced) on every release to broadcast to waiters;
- // each re-checks against the current limit, so a raised limit can admit
- // more than one of them.
- wake chan struct{}
-}
-
-func newActivitySlotLimiterInternal(localSettings *settings.SettingsService) *activitySlotLimiter {
- return &activitySlotLimiter{
- settings: localSettings,
- state: map[string]*activityEnvironmentSlots{},
- }
-}
-
-func (l *activitySlotLimiter) limitInternal(ctx context.Context) int {
- return l.settings.GetIntSetting(ctx, maxConcurrentActivitiesSettingKey, defaultMaxConcurrentActivities)
-}
-
-func (l *activitySlotLimiter) stateForLockedInternal(environmentID string) *activityEnvironmentSlots {
- slots := l.state[environmentID]
- if slots == nil {
- slots = &activityEnvironmentSlots{wake: make(chan struct{})}
- l.state[environmentID] = slots
- }
- return slots
-}
-
-// tryAcquireInternal grabs a slot without blocking; None means the caller
-// should queue and block via acquireInternal. Held slots are counted even
-// while the limit is unlimited, so enabling a limit later still sees the work
-// already running.
-func (l *activitySlotLimiter) tryAcquireInternal(ctx context.Context, environmentID string) mo.Option[func()] {
- if l == nil || l.settings == nil {
- return mo.Some(func() {})
- }
- limit := l.limitInternal(ctx)
-
- l.mu.Lock()
- defer l.mu.Unlock()
- slots := l.stateForLockedInternal(environmentID)
- if limit > 0 && slots.held >= limit {
- return mo.None[func()]()
- }
- slots.held++
- return mo.Some(l.releaseOnceInternal(environmentID))
-}
-
-// acquireInternal blocks until a slot frees or ctx is cancelled. The limit is
-// re-read on every attempt so setting changes apply to queued waiters.
-func (l *activitySlotLimiter) acquireInternal(ctx context.Context, environmentID string) (func(), error) {
- if l == nil || l.settings == nil {
- return func() {}, nil
- }
-
- for {
- limit := l.limitInternal(ctx)
- l.mu.Lock()
- slots := l.stateForLockedInternal(environmentID)
- if limit <= 0 || slots.held < limit {
- slots.held++
- l.mu.Unlock()
- return l.releaseOnceInternal(environmentID), nil
- }
- wake := slots.wake
- l.mu.Unlock()
-
- timer := time.NewTimer(slotWaitRecheckInterval)
- select {
- case <-wake:
- case <-timer.C:
- case <-ctx.Done():
- timer.Stop()
- return nil, context.Cause(ctx)
- }
- timer.Stop()
- }
-}
-
-func (l *activitySlotLimiter) releaseOnceInternal(environmentID string) func() {
- var once sync.Once
- return func() {
- once.Do(func() {
- l.mu.Lock()
- slots := l.stateForLockedInternal(environmentID)
- if slots.held > 0 {
- slots.held--
- }
- close(slots.wake)
- slots.wake = make(chan struct{})
- l.mu.Unlock()
- })
- }
-}
-
// Job summaries live in their target environment's environment_id column.
-func scopeJobActivityVisibilityInternal(ctx context.Context, query *gorm.DB, environmentID, permission string) *gorm.DB {
+func scopeJobActivityVisibility(ctx context.Context, query *gorm.DB, environmentID, permission string) *gorm.DB {
permissions, present := middleware.PermissionsFromContext(ctx)
if !present || permissions.Allows(permission, environmentID) {
return query
@@ -140,7 +20,7 @@ func scopeJobActivityVisibilityInternal(ctx context.Context, query *gorm.DB, env
return query.Where("type <> ?", activity.TypeJobRun)
}
-func canReadJobActivityInternal(ctx context.Context, item activity.Activity) bool {
+func canReadJobActivity(ctx context.Context, item activity.Activity) bool {
if item.Type != activity.TypeJobRun {
return true
}
@@ -153,9 +33,9 @@ func canReadJobActivityInternal(ctx context.Context, item activity.Activity) boo
return permissions.Allows(authz.PermActivitiesRead, environmentID)
}
-func (h *ActivityHandler) canReadActivityStreamEventInternal(ctx context.Context, event activity.StreamEvent) bool {
+func (h *ActivityHandler) canReadActivityStreamEvent(ctx context.Context, event activity.StreamEvent) bool {
if event.Activity != nil {
- return canReadJobActivityInternal(ctx, *event.Activity)
+ return canReadJobActivity(ctx, *event.Activity)
}
if event.ActivityID == "" {
return true
@@ -171,5 +51,5 @@ func (h *ActivityHandler) canReadActivityStreamEventInternal(ctx context.Context
if err := h.activityService.db.WithContext(ctx).Select("type", "environment_id", "metadata").Where("id = ? AND environment_id = ?", event.ActivityID, "0").First(&model).Error; err != nil {
return false
}
- return canReadJobActivityInternal(ctx, activity.Activity{Type: model.Type, EnvironmentID: model.EnvironmentID, Metadata: model.Metadata})
+ return canReadJobActivity(ctx, activity.Activity{Type: model.Type, EnvironmentID: model.EnvironmentID, Metadata: model.Metadata})
}
diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.go
index d229abe88..df1cea574 100644
--- a/backend/internal/activity/service.go
+++ b/backend/internal/activity/service.go
@@ -21,6 +21,9 @@ import (
"gorm.io/gorm"
"gorm.io/gorm/clause"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/pubsub"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/slots"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/sweeps"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
@@ -36,29 +39,8 @@ const (
defaultActivityRetentionDays = 30
defaultActivityHistoryLimit = 1000
defaultActivityMessages = 500
- staleImageUpdateCheckAge = 6 * time.Hour
- // abandonedActivityGrace is how old an untracked queued/running activity
- // must be before the periodic sweep fails it. It covers the window between
- // StartActivity's row creation and the worker's Track call.
- abandonedActivityGrace = 2 * time.Minute
- // abandonedActivitySweepLimit bounds one sweep; later sweeps take the rest.
- abandonedActivitySweepLimit = 500
-
- // terminalPublishRetention bounds how long a terminal publish suppresses
- // stale non-terminal snapshots for the same activity. Stale publishers are
- // goroutines already past their commit, so they publish within moments; the
- // retention only has to outlive that gap while keeping the latch map small.
- terminalPublishRetention = 10 * time.Minute
-
- // ErrActivityNotCancelable indicates the activity has already reached a terminal
- // state and can no longer be cancelled.
-
- // subscriberMessageQueueLimit bounds the per-subscriber backlog of "message"
- // events; the oldest message is dropped (and flagged as missed) on overflow.
- subscriberMessageQueueLimit = 256
-
- cancelledMessageInternal = "Cancelled by user"
+ cancelledMessage = "Cancelled by user"
deleteActivitiesBatchSize = 500
)
@@ -66,11 +48,10 @@ const (
var ErrActivityNotCancelable = errors.New("activity is not cancelable")
type ActivityService struct {
- db *database.DB
-
- subscribersMu sync.RWMutex
- subscribers map[int]*activitySubscriber
- nextSubID int
+ db *database.DB
+ pubsub *pubsub.Service
+ slots *slots.Service
+ sweeps *sweeps.Service
// running maps an active activity ID to the cancel function of its work
// context, so cancellation requests can interrupt in-flight work. Entries
@@ -78,149 +59,16 @@ type ActivityService struct {
runningMu sync.Mutex
running map[string]context.CancelCauseFunc
- // limiter bounds concurrent queue-opted activities per environment.
- // slotReleases maps an activity ID to the release func of the slot it
- // holds; the slot is freed when the activity completes.
- limiter *activitySlotLimiter
- slotMu sync.Mutex
- slotReleases map[string]func()
-
- // terminalPublished records when each activity's terminal snapshot was
- // published, so later non-terminal snapshots from slower goroutines can
- // be dropped (see admitActivityPublishInternal). Entries are pruned on
- // terminal publishes once older than terminalPublishRetention.
- terminalPublishedMu sync.Mutex
- terminalPublished map[string]time.Time
-
// publishLocks serializes the commit→publish window of the non-terminal
// snapshot writers (AppendMessages, UpdateActivity) per activity, so their
// events reach subscribers in commit order — without it, a batch that
// commits first but publishes second would revert progress/step/status
// fields a concurrent update already streamed. Terminal snapshots are
- // ordered separately by admitActivityPublishInternal. Striped by activity
+ // ordered separately by the pubsub terminal latch. Striped by activity
// ID hash so no per-activity lifecycle management is needed.
publishLocks [64]sync.Mutex
}
-// activitySubscriber buffers stream events between publishers and one stream
-// consumer. "activity" events are coalesced in place per activity ID (only the
-// latest pending state matters to the UI), so bulk operations emitting rapid
-// progress updates cannot overflow the subscriber and force full-snapshot
-// resends. Other events keep arrival order in a FIFO bounded by
-// subscriberMessageQueueLimit with drop-oldest on overflow.
-type activitySubscriber struct {
- environmentID string
- ch chan activity.StreamEvent
- done chan struct{}
- wake chan struct{}
-
- mu sync.Mutex
- missed bool
- queue []*pendingStreamEvent
- pendingActivity map[string]*pendingStreamEvent
- messageCount int
-}
-
-type pendingStreamEvent struct {
- event activity.StreamEvent
-}
-
-func newActivitySubscriberInternal(environmentID string, ch chan activity.StreamEvent) *activitySubscriber {
- return &activitySubscriber{
- environmentID: environmentID,
- ch: ch,
- done: make(chan struct{}),
- wake: make(chan struct{}, 1),
- pendingActivity: map[string]*pendingStreamEvent{},
- }
-}
-
-func isCoalescableEventInternal(event activity.StreamEvent) bool {
- return event.Type == "activity" && event.ActivityID != ""
-}
-
-func (sub *activitySubscriber) enqueue(event activity.StreamEvent) {
- sub.mu.Lock()
- if isCoalescableEventInternal(event) {
- if pending, ok := sub.pendingActivity[event.ActivityID]; ok {
- pending.event = event
- sub.mu.Unlock()
- return
- }
- } else {
- if sub.messageCount >= subscriberMessageQueueLimit {
- sub.dropOldestMessageLockedInternal()
- }
- sub.messageCount++
- }
- entry := &pendingStreamEvent{event: event}
- sub.queue = append(sub.queue, entry)
- if isCoalescableEventInternal(event) {
- sub.pendingActivity[event.ActivityID] = entry
- }
- sub.mu.Unlock()
-
- select {
- case sub.wake <- struct{}{}:
- default:
- }
-}
-
-func (sub *activitySubscriber) dropOldestMessageLockedInternal() {
- for i, entry := range sub.queue {
- if isCoalescableEventInternal(entry.event) {
- continue
- }
- sub.queue = append(sub.queue[:i], sub.queue[i+1:]...)
- sub.messageCount--
- sub.missed = true
- ctx := context.Background() //nolint:forbidigo // Subscriber buffer trimming runs outside any request context.
- slog.WarnContext(ctx, "activity subscriber message buffer full; snapshot will be sent on next heartbeat",
- "environmentId", sub.environmentID)
- return
- }
-}
-
-func (sub *activitySubscriber) nextInternal() mo.Option[activity.StreamEvent] {
- sub.mu.Lock()
- defer sub.mu.Unlock()
-
- if len(sub.queue) == 0 {
- return mo.None[activity.StreamEvent]()
- }
- entry := sub.queue[0]
- sub.queue = sub.queue[1:]
- event := entry.event
- if isCoalescableEventInternal(event) {
- if sub.pendingActivity[event.ActivityID] == entry {
- delete(sub.pendingActivity, event.ActivityID)
- }
- } else {
- sub.messageCount--
- }
- return mo.Some(event)
-}
-
-func (sub *activitySubscriber) pump() {
- defer close(sub.ch)
- for {
- event, ok := sub.nextInternal().Get()
- if !ok {
- select {
- case <-sub.wake:
- continue
- case <-sub.done:
- return
- }
- }
- select {
- case sub.ch <- event:
- case <-sub.done:
- return
- }
- }
-}
-
type (
StartActivityRequest = activitylib.StartRequest
UpdateActivityRequest = activitylib.UpdateRequest
@@ -229,12 +77,11 @@ type (
func NewActivityService(db *database.DB, settingsService *settings.SettingsService) *ActivityService {
return &ActivityService{
- db: db,
- subscribers: map[int]*activitySubscriber{},
- running: map[string]context.CancelCauseFunc{},
- limiter: newActivitySlotLimiterInternal(settingsService),
- slotReleases: map[string]func(){},
- terminalPublished: map[string]time.Time{},
+ db: db,
+ pubsub: pubsub.NewService(),
+ slots: slots.NewService(settingsService),
+ sweeps: sweeps.NewService(db),
+ running: map[string]context.CancelCauseFunc{},
}
}
@@ -281,27 +128,7 @@ func (s *ActivityService) RequestCancel(activityID string) bool {
return true
}
-// releaseCancelInternal removes and cancels the registration for activityID.
-// Cancelling with a nil cause is a no-op if the context was already cancelled
-// (the first cause wins), so a prior ErrCanceled cause is preserved.
-func (s *ActivityService) releaseCancelInternal(activityID string) {
- activityID = strings.TrimSpace(activityID)
- if s == nil || activityID == "" {
- return
- }
-
- s.runningMu.Lock()
- cancel, ok := s.running[activityID]
- if ok {
- delete(s.running, activityID)
- }
- s.runningMu.Unlock()
- if ok {
- cancel(nil)
- }
-}
-
-func (s *ActivityService) checkInitInternal() error {
+func (s *ActivityService) checkInit() error {
if s == nil || s.db == nil {
return errors.New("activity service not initialized")
}
@@ -309,7 +136,7 @@ func (s *ActivityService) checkInitInternal() error {
}
func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRequest) (*activity.Activity, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, err
}
@@ -320,7 +147,7 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRe
return nil, fmt.Errorf("failed to load activity: %w", err)
}
if existing.ID != "" {
- dto := activityToDTOInternal(&existing)
+ dto := activityToDTO(&existing)
return &dto, nil
}
}
@@ -328,22 +155,6 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRe
now := time.Now()
environmentID := cmp.Or(strings.TrimSpace(req.EnvironmentID), "0")
- var startedByUserID, startedByUsername, startedByDisplayName *string
- if req.StartedBy != nil {
- startedByUserID = mo.EmptyableToOption(strings.TrimSpace(req.StartedBy.ID)).ToPointer()
- startedByUsername = mo.EmptyableToOption(strings.TrimSpace(req.StartedBy.Username)).ToPointer()
- if req.StartedBy.DisplayName != nil {
- startedByDisplayName = mo.EmptyableToOption(strings.TrimSpace(*req.StartedBy.DisplayName)).ToPointer()
- }
- }
-
- batchID := req.BatchID
- if batchID == nil {
- if contextBatchID := utils.ActivityBatchIDFromContext(ctx); contextBatchID != "" {
- batchID = &contextBatchID
- }
- }
-
// Queue-opted activities take a concurrency slot up front when one is
// free; otherwise they are created as queued and AwaitActivitySlot blocks
// until a slot opens.
@@ -352,35 +163,33 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRe
if req.Queue {
if req.DeferSlot {
status = activity.StatusQueued
- } else if release, ok := s.limiter.tryAcquireInternal(ctx, environmentID).Get(); ok {
+ } else if release, ok := s.slots.TryAcquire(ctx, environmentID).Get(); ok {
slotRelease = release
} else {
status = activity.StatusQueued
}
}
+ startedBy := kit.FromPtr(req.StartedBy)
model := &Activity{
+ ID: req.ID,
EnvironmentID: environmentID,
- BatchID: copyPtrInternal(batchID),
- Type: req.Type,
+ BatchID: copyPtr(kit.Ternary(req.BatchID != nil, req.BatchID, mo.EmptyableToOption(utils.ActivityBatchIDFromContext(ctx)).ToPointer())),
+ Type: cmp.Or(req.Type, activity.TypeAutoUpdate),
Status: status,
- ResourceType: copyPtrInternal(req.ResourceType),
- ResourceID: copyPtrInternal(req.ResourceID),
- ResourceName: copyPtrInternal(req.ResourceName),
- StartedByUserID: startedByUserID,
- StartedByUsername: startedByUsername,
- StartedByDisplayName: startedByDisplayName,
- Progress: clampProgressPtrInternal(req.Progress),
+ ResourceType: copyPtr(req.ResourceType),
+ ResourceID: copyPtr(req.ResourceID),
+ ResourceName: copyPtr(req.ResourceName),
+ StartedByUserID: mo.EmptyableToOption(strings.TrimSpace(startedBy.ID)).ToPointer(),
+ StartedByUsername: mo.EmptyableToOption(strings.TrimSpace(startedBy.Username)).ToPointer(),
+ StartedByDisplayName: mo.EmptyableToOption(strings.TrimSpace(kit.FromPtr(startedBy.DisplayName))).ToPointer(),
+ Progress: clampProgressPtr(req.Progress),
Step: strings.TrimSpace(req.Step),
LatestMessage: strings.TrimSpace(req.LatestMessage),
StartedAt: now,
- Metadata: cloneJSONInternal(req.Metadata),
+ Metadata: cloneJSON(req.Metadata),
CreatedAt: now,
}
- model.ID = req.ID
- if model.Type == "" {
- model.Type = activity.TypeAutoUpdate
- }
create := s.db.WithContext(ctx)
if req.ID != "" {
@@ -399,22 +208,22 @@ func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRe
if err := s.db.WithContext(ctx).First(&existing, "id = ?", req.ID).Error; err != nil {
return nil, fmt.Errorf("failed to load activity: %w", err)
}
- dto := activityToDTOInternal(&existing)
+ dto := activityToDTO(&existing)
return &dto, nil
}
if slotRelease != nil {
- s.registerSlotReleaseInternal(model.ID, slotRelease)
+ s.slots.Register(model.ID, slotRelease)
}
- dto := activityToDTOInternal(model)
- s.publishActivityInternal(dto)
+ dto := activityToDTO(model)
+ s.pubsub.PublishActivity(dto)
return &dto, nil
}
// StartTrackedActivity holds the cancellation lock across persistence and
// registration, so cancellation cannot treat the new row as untracked.
func (s *ActivityService) StartTrackedActivity(ctx context.Context, req StartActivityRequest) (*activity.Activity, context.Context, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, nil, err
}
s.runningMu.Lock()
@@ -431,37 +240,13 @@ func (s *ActivityService) StartTrackedActivity(ctx context.Context, req StartAct
return started, workCtx, nil
}
-func (s *ActivityService) registerSlotReleaseInternal(activityID string, release func()) {
- s.slotMu.Lock()
- if existing, ok := s.slotReleases[activityID]; ok {
- existing()
- }
- s.slotReleases[activityID] = release
- s.slotMu.Unlock()
-}
-
-func (s *ActivityService) releaseSlotInternal(activityID string) {
- if s == nil {
- return
- }
- s.slotMu.Lock()
- release, ok := s.slotReleases[activityID]
- if ok {
- delete(s.slotReleases, activityID)
- }
- s.slotMu.Unlock()
- if ok {
- release()
- }
-}
-
// AwaitActivitySlot blocks until the queued activity holds a concurrency slot,
// then flips its status to running. It returns immediately when the activity
// already took a slot at creation. On cancellation the context cause is
// returned and the activity stays queued for its caller to finalize.
// Implements activitylib.SlotWaiter.
func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, environmentID string) error {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return err
}
activityID = strings.TrimSpace(activityID)
@@ -470,18 +255,10 @@ func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, env
}
environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
- s.slotMu.Lock()
- _, held := s.slotReleases[activityID]
- s.slotMu.Unlock()
- if held {
- return nil
- }
-
- release, err := s.limiter.acquireInternal(ctx, environmentID)
- if err != nil {
+ acquired, err := s.slots.Await(ctx, activityID, environmentID)
+ if err != nil || !acquired {
return err
}
- s.registerSlotReleaseInternal(activityID, release)
if _, updateErr := s.UpdateActivity(ctx, activityID, UpdateActivityRequest{Status: activity.StatusRunning}); updateErr != nil {
slog.WarnContext(ctx, "failed to mark queued activity running", "activityId", activityID, "error", updateErr)
@@ -506,7 +283,7 @@ func (s *ActivityService) AwaitActivitySlotBounded(ctx context.Context, activity
}
func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string, req UpdateActivityRequest) (*activity.Activity, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, err
}
activityID = strings.TrimSpace(activityID)
@@ -524,7 +301,7 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,
updates["resource_name"] = strings.TrimSpace(*req.ResourceName)
}
if req.Progress != nil {
- updates["progress"] = *clampProgressPtrInternal(req.Progress)
+ updates["progress"] = *clampProgressPtr(req.Progress)
}
if req.Step != nil {
updates["step"] = strings.TrimSpace(*req.Step)
@@ -536,13 +313,13 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,
updates["error"] = strings.TrimSpace(*req.Error)
}
if req.Metadata != nil {
- updates["metadata"] = cloneJSONInternal(req.Metadata)
+ updates["metadata"] = cloneJSON(req.Metadata)
}
// Hold the per-activity publish lock across commit and publish so this
// snapshot cannot be overtaken on the stream by a concurrent append batch
// that committed earlier but would publish later.
- lock := s.publishLockInternal(activityID)
+ lock := s.publishLock(activityID)
lock.Lock()
defer lock.Unlock()
@@ -563,8 +340,8 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,
return nil, err
}
- dto := activityToDTOInternal(&model)
- s.publishActivityInternal(dto)
+ dto := activityToDTO(&model)
+ s.pubsub.PublishActivity(dto)
return &dto, nil
}
@@ -584,9 +361,9 @@ func (s *ActivityService) AppendMessage(ctx context.Context, activityID string,
// per batch, not per line) so it reflects lifecycle fields a concurrent
// CompleteActivity/UpdateActivity committed first; a terminal write that
// commits after this transaction but publishes before this snapshot is
-// handled by admitActivityPublishInternal dropping the stale event.
+// handled by the pubsub terminal latch dropping the stale event.
func (s *ActivityService) AppendMessages(ctx context.Context, activityID string, reqs []AppendActivityMessageRequest) ([]activity.Message, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, err
}
activityID = strings.TrimSpace(activityID)
@@ -594,7 +371,7 @@ func (s *ActivityService) AppendMessages(ctx context.Context, activityID string,
return nil, errors.New("activity id is required")
}
- messages, updates := buildAppendBatchInternal(activityID, reqs)
+ messages, updates := buildAppendBatch(activityID, reqs)
if len(messages) == 0 {
return nil, nil
}
@@ -603,29 +380,58 @@ func (s *ActivityService) AppendMessages(ctx context.Context, activityID string,
// batch's snapshot cannot be overtaken on the stream by a concurrent
// UpdateActivity that committed later — subscribers must see the two
// snapshots in commit order.
- lock := s.publishLockInternal(activityID)
+ lock := s.publishLock(activityID)
lock.Lock()
defer lock.Unlock()
- current, err := s.appendBatchWithRetryInternal(ctx, activityID, messages, updates)
- if err != nil {
+ // The Writer drain cannot replay a lost batch, so retry like CompleteActivity.
+ var current Activity
+ if err := s.retryWrite(ctx, activityID, "message batch", func(tx *gorm.DB) error {
+ if err := tx.First(&current, "id = ?", activityID).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.New("activity not found")
+ }
+ return fmt.Errorf("failed to load activity: %w", err)
+ }
+ if err := tx.Create(&messages).Error; err != nil {
+ return fmt.Errorf("failed to append activity message: %w", err)
+ }
+ result := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates)
+ if result.Error != nil {
+ return fmt.Errorf("failed to update activity latest message: %w", result.Error)
+ }
+ if result.RowsAffected == 0 {
+ return errors.New("activity not found")
+ }
+ if err := tx.First(&current, "id = ?", activityID).Error; err != nil {
+ return fmt.Errorf("failed to load updated activity: %w", err)
+ }
+ return nil
+ }); err != nil {
return nil, err
}
out := make([]activity.Message, 0, len(messages))
for _, message := range messages {
- dto := activityMessageToDTOInternal(message)
- s.publishMessageInternal(current.EnvironmentID, current.Type, dto)
+ dto := activity.Message{
+ ID: message.ID,
+ ActivityID: message.ActivityID,
+ Level: message.Level,
+ Message: message.Message,
+ Payload: cloneJSON(message.Payload),
+ CreatedAt: message.CreatedAt,
+ }
+ s.pubsub.PublishMessage(current.EnvironmentID, current.Type, dto)
out = append(out, dto)
}
- s.publishActivityInternal(activityToDTOInternal(current))
+ s.pubsub.PublishActivity(activityToDTO(&current))
return out, nil
}
-// buildAppendBatchInternal turns the request batch into insertable message rows
+// buildAppendBatch turns the request batch into insertable message rows
// plus the coalesced Activity column updates (latest message/progress/step from
// the last request carrying each).
-func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageRequest) ([]*ActivityMessage, map[string]any) {
+func buildAppendBatch(activityID string, reqs []AppendActivityMessageRequest) ([]*ActivityMessage, map[string]any) {
now := time.Now()
messages := make([]*ActivityMessage, 0, len(reqs))
latestMessage := ""
@@ -646,7 +452,7 @@ func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageReq
ActivityID: activityID,
Level: level,
Message: messageText,
- Payload: cloneJSONInternal(req.Payload),
+ Payload: cloneJSON(req.Payload),
// Spread inside the shared timestamp so the created_at sort
// used for retrieval keeps the original line order.
CreatedAt: now.Add(time.Duration(len(messages)) * time.Microsecond),
@@ -669,7 +475,7 @@ func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageReq
"updated_at": now,
}
if latestProgress != nil {
- updates["progress"] = *clampProgressPtrInternal(latestProgress)
+ updates["progress"] = *clampProgressPtr(latestProgress)
}
if latestStep != "" {
updates["step"] = latestStep
@@ -677,70 +483,25 @@ func buildAppendBatchInternal(activityID string, reqs []AppendActivityMessageReq
return messages, updates
}
-// appendBatchWithRetryInternal commits the batch INSERT plus Activity update in
-// one transaction. Bulk appends contend with terminal-status writes for the
-// SQLite write lock, and a batch lost to transient SQLITE_BUSY silently drops
-// up to 32 output lines — the Writer drain has no error path to replay them —
-// so retry like CompleteActivity does.
-func (s *ActivityService) appendBatchWithRetryInternal(ctx context.Context, activityID string, messages []*ActivityMessage, updates map[string]any) (*Activity, error) {
- var current Activity
- const appendWriteAttempts = 3
- var writeErr error
- for attempt := 1; attempt <= appendWriteAttempts; attempt++ {
- writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if err := tx.First(&current, "id = ?", activityID).Error; err != nil {
- if errors.Is(err, gorm.ErrRecordNotFound) {
- return errors.New("activity not found")
- }
- return fmt.Errorf("failed to load activity: %w", err)
- }
-
- if err := tx.Create(&messages).Error; err != nil {
- return fmt.Errorf("failed to append activity message: %w", err)
- }
-
- result := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates)
- if result.Error != nil {
- return fmt.Errorf("failed to update activity latest message: %w", result.Error)
- }
- if result.RowsAffected == 0 {
- return errors.New("activity not found")
- }
- if err := tx.First(&current, "id = ?", activityID).Error; err != nil {
- return fmt.Errorf("failed to load updated activity: %w", err)
- }
- return nil
- })
- if writeErr == nil {
- return &current, nil
- }
- if attempt == appendWriteAttempts || !dbutil.IsRetryableWriteError(writeErr) {
- return nil, writeErr
+// retryWrite runs a write transaction, retrying transient SQLite contention
+// (SQLITE_BUSY under bulk activity writes) up to three times.
+func (s *ActivityService) retryWrite(ctx context.Context, activityID, what string, write func(tx *gorm.DB) error) error {
+ const attempts = 3
+ for attempt := 1; ; attempt++ {
+ err := s.db.WithContext(ctx).Transaction(write)
+ if err == nil || attempt == attempts || !dbutil.IsRetryableWriteError(err) {
+ return err
}
- slog.WarnContext(ctx, "retrying activity message batch write", "activityId", activityID, "attempt", attempt, "error", writeErr)
+ slog.WarnContext(ctx, "retrying activity write", "write", what, "activityId", activityID, "attempt", attempt, "error", err)
time.Sleep(250 * time.Millisecond * time.Duration(attempt))
}
- return nil, writeErr
}
-func (
- s *ActivityService,
-) CompleteActivity(
- ctx context.Context,
- activityID string,
- status activity.Status,
- finalMessage string,
- errMessage *string,
- finalStep ...string,
-) (
- *activity.Activity,
- error,
-) {
- if err := s.checkInitInternal(); err != nil {
+func (s *ActivityService) CompleteActivity(ctx context.Context, activityID string, status activity.Status, finalMessage string, errMessage *string, finalStep ...string) (*activity.Activity, error) {
+ if err := s.checkInit(); err != nil {
return nil, err
}
- status = cmp.Or(status, activity.StatusSuccess)
- if status != activity.StatusSuccess && status != activity.StatusFailed && status != activity.StatusCancelled {
+ if !slices.Contains(terminalActivityStatuses, status) {
status = activity.StatusSuccess
}
@@ -750,9 +511,15 @@ func (
}
// The activity is reaching a terminal state; release any cancel
- // registration and free its concurrency slot.
- s.releaseCancelInternal(activityID)
- s.releaseSlotInternal(activityID)
+ // registration (a nil cause keeps an earlier ErrCanceled) and free its slot.
+ s.runningMu.Lock()
+ cancel, tracked := s.running[activityID]
+ delete(s.running, activityID)
+ s.runningMu.Unlock()
+ if tracked {
+ cancel(nil)
+ }
+ s.slots.Release(activityID)
// Detach from cancellation so the terminal write always lands — completion is
// often triggered precisely because the work context was cancelled.
@@ -760,37 +527,22 @@ func (
now := time.Now()
var model Activity
- // A lost terminal write leaves the activity stuck in running forever, so
- // retry transient DB contention (SQLITE_BUSY under bulk activity writes)
- // instead of surfacing it once and giving up.
- const completeWriteAttempts = 3
- var writeErr error
- for attempt := 1; attempt <= completeWriteAttempts; attempt++ {
- writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
- return fmt.Errorf("failed to load activity: %w", err)
- }
+ // A lost terminal write leaves the activity stuck in running forever.
+ if err := s.retryWrite(ctx, activityID, "terminal status", func(tx *gorm.DB) error {
+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
+ return fmt.Errorf("failed to load activity: %w", err)
+ }
- updates := completeActivityUpdatesInternal(model.StartedAt, status, finalMessage, errMessage, finalStep, now)
- if err := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {
- return fmt.Errorf("failed to complete activity: %w", err)
- }
- if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
- return fmt.Errorf("failed to load completed activity: %w", err)
- }
- return nil
- })
- if writeErr == nil {
- break
+ updates := completeActivityUpdates(model.StartedAt, status, finalMessage, errMessage, finalStep, now)
+ if err := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {
+ return fmt.Errorf("failed to complete activity: %w", err)
}
- if attempt == completeWriteAttempts || !dbutil.IsRetryableWriteError(writeErr) {
- return nil, writeErr
+ if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
+ return fmt.Errorf("failed to load completed activity: %w", err)
}
- slog.WarnContext(ctx, "retrying activity terminal status write", "activityId", activityID, "attempt", attempt, "error", writeErr)
- time.Sleep(250 * time.Millisecond * time.Duration(attempt))
- }
- if writeErr != nil {
- return nil, writeErr
+ return nil
+ }); err != nil {
+ return nil, err
}
if strings.TrimSpace(finalMessage) != "" {
@@ -811,11 +563,11 @@ func (
}
}
- dto := s.publishTerminalSnapshotInternal(ctx, &model)
+ dto := s.publishTerminalSnapshot(ctx, &model)
return &dto, nil
}
-// publishTerminalSnapshotInternal publishes an activity's terminal event from
+// publishTerminalSnapshot publishes an activity's terminal event from
// a snapshot re-read under the per-activity publish lock. Terminal writers
// cannot hold that lock across their own transactions (CompleteActivity
// appends its final message through the locked AppendMessages path), so a
@@ -824,8 +576,8 @@ func (
// guarantees the terminal event carries every field already streamed. The
// passed model is published as-is if the re-read fails, and is updated in
// place otherwise so callers return the published state.
-func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, model *Activity) activity.Activity {
- lock := s.publishLockInternal(model.ID)
+func (s *ActivityService) publishTerminalSnapshot(ctx context.Context, model *Activity) activity.Activity {
+ lock := s.publishLock(model.ID)
lock.Lock()
defer lock.Unlock()
@@ -835,8 +587,8 @@ func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, m
} else {
*model = fresh
}
- dto := activityToDTOInternal(model)
- s.publishActivityInternal(dto)
+ dto := activityToDTO(model)
+ s.pubsub.PublishActivity(dto)
return dto
}
@@ -846,7 +598,7 @@ func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, m
// but only if it is still active. Returns ErrActivityNotCancelable if the activity
// has already reached a terminal state, or gorm.ErrRecordNotFound if it is unknown.
func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, activityID, requestedBy string) (*activity.Activity, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, err
}
activityID = strings.TrimSpace(activityID)
@@ -883,7 +635,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act
// terminal status, which reaches clients via the activity stream. Return
// the pre-cancel snapshot rather than reloading here: the worker has not
// finished unwinding yet, so a reload would still report "running".
- return new(activityToDTOInternal(&model)), nil
+ return new(activityToDTO(&model)), nil
}
// Untracked work (e.g. after a process restart, or a queued activity with no
@@ -895,7 +647,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act
if err := tx.First(&finalized, "id = ? AND environment_id = ?", activityID, environmentID).Error; err != nil {
return err
}
- updates := completeActivityUpdatesInternal(finalized.StartedAt, activity.StatusCancelled, cancelledMessageInternal, nil, nil, now)
+ updates := completeActivityUpdates(finalized.StartedAt, activity.StatusCancelled, cancelledMessage, nil, nil, now)
result := tx.Model(&Activity{}).
Where("id = ? AND status IN ?", activityID, []activity.Status{activity.StatusQueued, activity.StatusRunning}).
Updates(updates)
@@ -913,7 +665,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act
return nil, err
}
- dto := s.publishTerminalSnapshotInternal(writeCtx, &finalized)
+ dto := s.publishTerminalSnapshot(writeCtx, &finalized)
return &dto, nil
}
@@ -921,40 +673,40 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act
// across a prior process lifetime as failed. It intentionally scopes cleanup to
// old image-update-check activities so startup repair cannot affect other work.
func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64, error) {
- if s == nil || s.db == nil {
- return 0, nil
- }
+ return s.finalizeSwept(ctx, "stale image update check",
+ func() ([]sweeps.Candidate, error) { return s.sweeps.StaleImageUpdateChecks(ctx) },
+ func(sweeps.Candidate) (activity.Status, string) {
+ return activity.StatusFailed, "Image update check failed because it was stale after Arcane restarted"
+ },
+ "Image update check failed",
+ )
+}
- cutoff := time.Now().Add(-staleImageUpdateCheckAge)
- var staleChecks []Activity
- if err := s.db.WithContext(ctx).
- Where("type = ? AND status = ? AND started_at < ?", activity.TypeImageUpdateCheck, activity.StatusRunning, cutoff).
- Find(&staleChecks).Error; err != nil {
- return 0, fmt.Errorf("find stale image update checks: %w", err)
- }
-
- const message = "Image update check failed because it was stale after Arcane restarted"
- errMessage := message
- var failed int64
- var failErrs []error
- for i := range staleChecks {
- if _, err := s.CompleteActivity(ctx, staleChecks[i].ID, activity.StatusFailed, message, &errMessage, "Image update check failed"); err != nil {
- failErrs = append(failErrs, fmt.Errorf("fail stale image update check %s: %w", staleChecks[i].ID, err))
+// finalizeSwept completes each swept candidate with the status and message
+// resolve picks; failed ones also record the message as their error.
+func (s *ActivityService) finalizeSwept(
+ ctx context.Context,
+ label string,
+ find func() ([]sweeps.Candidate, error),
+ resolve func(sweeps.Candidate) (activity.Status, string),
+ finalStep ...string,
+) (int64, error) {
+ candidates, err := find()
+ if err != nil {
+ return 0, fmt.Errorf("find %s activities: %w", label, err)
+ }
+ var finalized int64
+ var errs []error
+ for _, candidate := range candidates {
+ status, message := resolve(candidate)
+ errMessage := kit.Ternary(status == activity.StatusFailed, new(message), nil)
+ if _, completeErr := s.CompleteActivity(ctx, candidate.ID, status, message, errMessage, finalStep...); completeErr != nil {
+ errs = append(errs, fmt.Errorf("finalize %s activity %s: %w", label, candidate.ID, completeErr))
continue
}
- failed++
+ finalized++
}
-
- return failed, errors.Join(failErrs...)
-}
-
-// isTrackedInternal reports whether activityID has a live work registration in
-// this process (i.e. a worker goroutine called Track and has not completed yet).
-func (s *ActivityService) isTrackedInternal(activityID string) bool {
- s.runningMu.Lock()
- defer s.runningMu.Unlock()
- _, ok := s.running[activityID]
- return ok
+ return finalized, errors.Join(errs...)
}
// FailAbandonedActivities marks queued/running activities whose worker is no
@@ -970,27 +722,21 @@ func (s *ActivityService) isTrackedInternal(activityID string) bool {
// fallback: a worker completing concurrently wins the race and the row is
// skipped here.
func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, error) {
- if s == nil || s.db == nil {
- return 0, nil
- }
-
activeStatuses := []activity.Status{activity.StatusQueued, activity.StatusRunning}
s.runningMu.Lock()
trackedIDs := slices.Collect(maps.Keys(s.running))
s.runningMu.Unlock()
- query := s.db.WithContext(ctx).
- Select("id", "started_at").
- Where("status IN ? AND started_at < ?", activeStatuses, time.Now().Add(-abandonedActivityGrace)).
- Where("type <> ?", activity.TypeJobRun)
- if len(trackedIDs) > 0 {
- query = query.Where("id NOT IN ?", trackedIDs)
- }
- var candidates []Activity
- if err := query.Order("started_at").Limit(abandonedActivitySweepLimit).Find(&candidates).Error; err != nil {
- return 0, fmt.Errorf("find abandoned activities: %w", err)
+ candidates, findErr := s.sweeps.Abandoned(ctx, trackedIDs)
+ if findErr != nil {
+ return 0, fmt.Errorf("find abandoned activities: %w", findErr)
}
// A worker may have Tracked its activity since the snapshot above.
- candidates = slices.DeleteFunc(candidates, func(candidate Activity) bool { return s.isTrackedInternal(candidate.ID) })
+ s.runningMu.Lock()
+ candidates = slices.DeleteFunc(candidates, func(candidate sweeps.Candidate) bool {
+ _, tracked := s.running[candidate.ID]
+ return tracked
+ })
+ s.runningMu.Unlock()
if len(candidates) == 0 {
return 0, nil
}
@@ -1007,7 +753,7 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e
durationArgs = append(durationArgs, candidate.ID, now.Sub(candidate.StartedAt).Milliseconds())
}
durationExpr.WriteString(" ELSE duration_ms END")
- updates := completeActivityUpdatesInternal(now, activity.StatusFailed, message, new(message), nil, now)
+ updates := completeActivityUpdates(now, activity.StatusFailed, message, new(message), nil, now)
updates["duration_ms"] = gorm.Expr(durationExpr.String(), durationArgs...)
// Status-guarded so a worker completing concurrently wins; only rows this
@@ -1028,8 +774,8 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e
}
for i := range finalized {
- s.releaseSlotInternal(finalized[i].ID)
- s.publishTerminalSnapshotInternal(ctx, &finalized[i])
+ s.slots.Release(finalized[i].ID)
+ s.publishTerminalSnapshot(ctx, &finalized[i])
}
return int64(len(finalized)), nil
@@ -1039,38 +785,18 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e
// Queued state is owned by a live goroutine blocked on AwaitActivitySlot, so a
// queued row after a restart can never start running.
func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context, protectedIDs ...string) (int64, error) {
- if s == nil || s.db == nil {
- return 0, nil
- }
-
- var queued []Activity
- if err := s.db.WithContext(ctx).
- Where("status = ?", activity.StatusQueued).
- Where("type <> ?", activity.TypeJobRun).
- Where("id NOT IN ?", append(protectedIDs, "")).
- Find(&queued).Error; err != nil {
- return 0, fmt.Errorf("find orphaned queued activities: %w", err)
- }
-
- const message = "Queued activity was interrupted by an Arcane restart"
- errMessage := message
- var failed int64
- var failErrs []error
- for i := range queued {
- if _, err := s.CompleteActivity(ctx, queued[i].ID, activity.StatusFailed, message, &errMessage); err != nil {
- failErrs = append(failErrs, fmt.Errorf("fail orphaned queued activity %s: %w", queued[i].ID, err))
- continue
- }
- failed++
- }
-
- return failed, errors.Join(failErrs...)
+ return s.finalizeSwept(ctx, "orphaned queued",
+ func() ([]sweeps.Candidate, error) { return s.sweeps.OrphanedQueued(ctx, protectedIDs) },
+ func(sweeps.Candidate) (activity.Status, string) {
+ return activity.StatusFailed, "Queued activity was interrupted by an Arcane restart"
+ },
+ )
}
// PatchActivityMetadata merges patch into the activity's existing metadata,
// unlike UpdateActivity which replaces the metadata wholesale.
func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID string, patch database.JSON) error {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return err
}
activityID = strings.TrimSpace(activityID)
@@ -1086,7 +812,7 @@ func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID
if err := tx.First(&record, "id = ?", activityID).Error; err != nil {
return fmt.Errorf("failed to load activity: %w", err)
}
- merged := cloneJSONInternal(record.Metadata)
+ merged := cloneJSON(record.Metadata)
if merged == nil {
merged = database.JSON{}
}
@@ -1104,41 +830,18 @@ func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID
// self-update completed by restarting Arcane, so it is recorded as success;
// anything else still running at startup was interrupted and is failed.
func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context, protectedIDs ...string) (int64, error) {
- if s == nil || s.db == nil {
- return 0, nil
- }
-
- var stale []Activity
- if err := s.db.WithContext(ctx).
- Where("type = ? AND status = ?", activity.TypeAutoUpdate, activity.StatusRunning).
- Where("id NOT IN ?", append(protectedIDs, "")).
- Find(&stale).Error; err != nil {
- return 0, fmt.Errorf("find stale auto-update activities: %w", err)
- }
-
- var resolved int64
- var resolveErrs []error
- for i := range stale {
- status := activity.StatusFailed
- message := "Auto-update interrupted by Arcane restart"
- var errMessage *string
- if selfUpdate, _ := stale[i].Metadata["selfUpdateTriggered"].(bool); selfUpdate {
- status = activity.StatusSuccess
- message = "Auto-update completed — Arcane restarted with the updated image"
- } else {
- errMessage = new(message)
- }
- if _, err := s.CompleteActivity(ctx, stale[i].ID, status, message, errMessage); err != nil {
- resolveErrs = append(resolveErrs, fmt.Errorf("resolve stale auto-update activity %s: %w", stale[i].ID, err))
- continue
- }
- resolved++
- }
-
- return resolved, errors.Join(resolveErrs...)
+ return s.finalizeSwept(ctx, "stale auto-update",
+ func() ([]sweeps.Candidate, error) { return s.sweeps.StaleAutoUpdates(ctx, protectedIDs) },
+ func(candidate sweeps.Candidate) (activity.Status, string) {
+ if selfUpdate, _ := candidate.Metadata["selfUpdateTriggered"].(bool); selfUpdate {
+ return activity.StatusSuccess, "Auto-update completed — Arcane restarted with the updated image"
+ }
+ return activity.StatusFailed, "Auto-update interrupted by Arcane restart"
+ },
+ )
}
-func completeActivityUpdatesInternal(startedAt time.Time, status activity.Status, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {
+func completeActivityUpdates(startedAt time.Time, status activity.Status, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {
updates := map[string]any{
"status": status,
"ended_at": now,
@@ -1163,7 +866,7 @@ func completeActivityUpdatesInternal(startedAt time.Time, status activity.Status
}
func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]activity.Activity, pagination.Response, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, pagination.Response{}, err
}
@@ -1171,7 +874,7 @@ func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environme
var activities []Activity
q := s.db.WithContext(ctx).Model(&Activity{}).Where("environment_id = ?", environmentID)
- q = scopeJobActivityVisibilityInternal(ctx, q, environmentID, authz.PermActivitiesRead)
+ q = scopeJobActivityVisibility(ctx, q, environmentID, authz.PermActivitiesRead)
if term := strings.TrimSpace(params.Search); term != "" {
escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(term)
@@ -1204,13 +907,13 @@ func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environme
out := make([]activity.Activity, 0, len(activities))
for i := range activities {
- out = append(out, activityToDTOInternal(&activities[i]))
+ out = append(out, activityToDTO(&activities[i]))
}
return out, paginationResp, nil
}
func (s *ActivityService) GetActivityDetail(ctx context.Context, environmentID, activityID string, limit int) (*activity.Detail, error) {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return nil, err
}
if limit <= 0 || limit > defaultActivityMessages {
@@ -1234,20 +937,24 @@ func (s *ActivityService) GetActivityDetail(ctx context.Context, environmentID,
}
outMessages := make([]activity.Message, 0, len(messages))
- for _, v := range slices.Backward(messages) {
- outMessages = append(outMessages, activityMessageToDTOInternal(&v))
+ for _, message := range slices.Backward(messages) {
+ outMessages = append(outMessages, activity.Message{
+ ID: message.ID,
+ ActivityID: message.ActivityID,
+ Level: message.Level,
+ Message: message.Message,
+ Payload: cloneJSON(message.Payload),
+ CreatedAt: message.CreatedAt,
+ })
}
return &activity.Detail{
- Activity: activityToDTOInternal(&model),
+ Activity: activityToDTO(&model),
Messages: outMessages,
}, nil
}
func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEntries int) (int64, error) {
- if s == nil || s.db == nil {
- return 0, nil
- }
if retentionDays < 0 {
retentionDays = defaultActivityRetentionDays
}
@@ -1259,11 +966,13 @@ func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEn
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if retentionDays > 0 {
cutoff := time.Now().Add(-time.Duration(retentionDays) * 24 * time.Hour)
- ids, err := findTerminalActivityIDsInternal(tx.Where("COALESCE(ended_at, updated_at, created_at) < ?", cutoff))
- if err != nil {
+ var ids []string
+ if err := tx.Model(&Activity{}).
+ Where("status IN ? AND COALESCE(ended_at, updated_at, created_at) < ?", terminalActivityStatuses, cutoff).
+ Pluck("id", &ids).Error; err != nil {
return fmt.Errorf("failed to find activities older than retention window: %w", err)
}
- count, err := deleteActivitiesByIDInternal(tx, ids)
+ count, err := deleteActivitiesByID(tx, ids)
if err != nil {
return err
}
@@ -1271,11 +980,23 @@ func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEn
}
if maxEntries > 0 {
- ids, err := findActivityIDsBeyondHistoryLimitInternal(tx, maxEntries)
- if err != nil {
- return err
+ var ids []string
+ if err := tx.Raw(`
+ SELECT ranked.id
+ FROM (
+ SELECT id,
+ ROW_NUMBER() OVER (
+ PARTITION BY environment_id
+ ORDER BY COALESCE(ended_at, updated_at, created_at) DESC, started_at DESC
+ ) AS activity_rank
+ FROM activities
+ WHERE status IN ?
+ ) ranked
+ WHERE ranked.activity_rank > ?
+ `, terminalActivityStatuses, maxEntries).Scan(&ids).Error; err != nil {
+ return fmt.Errorf("failed to find excess activities: %w", err)
}
- count, err := deleteActivitiesByIDInternal(tx, ids)
+ count, err := deleteActivitiesByID(tx, ids)
if err != nil {
return err
}
@@ -1291,19 +1012,16 @@ func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEn
}
func (s *ActivityService) DeleteHistory(ctx context.Context, environmentID string) (int64, error) {
- if s == nil || s.db == nil {
- return 0, nil
- }
-
environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
var deleted int64
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- ids, err := findTerminalActivityIDsInternal(scopeJobActivityVisibilityInternal(ctx, tx.Where("environment_id = ?", environmentID), environmentID, authz.PermActivitiesDelete))
- if err != nil {
+ var ids []string
+ query := tx.Model(&Activity{}).Where("environment_id = ? AND status IN ?", environmentID, terminalActivityStatuses)
+ if err := scopeJobActivityVisibility(ctx, query, environmentID, authz.PermActivitiesDelete).Pluck("id", &ids).Error; err != nil {
return fmt.Errorf("failed to find activity history: %w", err)
}
- count, err := deleteActivitiesByIDInternal(tx, ids)
+ count, err := deleteActivitiesByID(tx, ids)
if err != nil {
return err
}
@@ -1316,147 +1034,13 @@ func (s *ActivityService) DeleteHistory(ctx context.Context, environmentID strin
return deleted, nil
}
-func (s *ActivityService) Subscribe(environmentID string) (<-chan activity.StreamEvent, func() bool, func()) {
- ch := make(chan activity.StreamEvent, 64)
- if s == nil {
- close(ch)
- return ch, func() bool { return false }, func() {}
- }
-
- environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
-
- sub := newActivitySubscriberInternal(environmentID, ch)
- s.subscribersMu.Lock()
- s.nextSubID++
- id := s.nextSubID
- s.subscribers[id] = sub
- s.subscribersMu.Unlock()
- go sub.pump()
-
- missedEvents := func() bool {
- s.subscribersMu.RLock()
- localSub, ok := s.subscribers[id]
- s.subscribersMu.RUnlock()
- if !ok {
- return false
- }
-
- localSub.mu.Lock()
- defer localSub.mu.Unlock()
- if !localSub.missed {
- return false
- }
- localSub.missed = false
- return true
- }
-
- unsubscribe := func() {
- s.subscribersMu.Lock()
- localSub, ok := s.subscribers[id]
- if ok {
- delete(s.subscribers, id)
- }
- s.subscribersMu.Unlock()
- if ok {
- // The pump goroutine owns ch and closes it on shutdown.
- close(localSub.done)
- }
- }
-
- return ch, missedEvents, unsubscribe
-}
-
-func (s *ActivityService) publishActivityInternal(item activity.Activity) {
- if !s.admitActivityPublishInternal(item) {
- return
- }
- s.publishInternal(item.EnvironmentID, activity.StreamEvent{
- Type: "activity",
- ActivityID: item.ID,
- Activity: &item,
- Timestamp: time.Now(),
- })
-}
-
-func (s *ActivityService) publishLockInternal(activityID string) *sync.Mutex {
+func (s *ActivityService) publishLock(activityID string) *sync.Mutex {
h := fnv.New32a()
_, _ = h.Write([]byte(activityID))
return &s.publishLocks[h.Sum32()%uint32(len(s.publishLocks))]
}
-func isTerminalActivityStatusInternal(status activity.Status) bool {
- return status == activity.StatusSuccess ||
- status == activity.StatusFailed ||
- status == activity.StatusCancelled
-}
-
-// admitActivityPublishInternal orders activity events across the commit →
-// publish gap. Writes publish their snapshot after committing, so a goroutine
-// that committed before a terminal write can publish after it; since a
-// terminal activity emits no further events to correct the stream (and
-// subscriber coalescing would even replace an undelivered terminal event in
-// place), such a stale non-terminal snapshot must be dropped, not delivered.
-// Activities never leave a terminal status, so once a terminal snapshot is
-// published, any later non-terminal snapshot for that ID is stale by
-// construction.
-func (s *ActivityService) admitActivityPublishInternal(item activity.Activity) bool {
- if s == nil {
- return true
- }
- s.terminalPublishedMu.Lock()
- defer s.terminalPublishedMu.Unlock()
- if !isTerminalActivityStatusInternal(item.Status) {
- _, sealed := s.terminalPublished[item.ID]
- return !sealed
- }
- now := time.Now()
- for id, publishedAt := range s.terminalPublished {
- if now.Sub(publishedAt) > terminalPublishRetention {
- delete(s.terminalPublished, id)
- }
- }
- if s.terminalPublished == nil {
- s.terminalPublished = map[string]time.Time{}
- }
- s.terminalPublished[item.ID] = now
- return true
-}
-
-func (s *ActivityService) publishMessageInternal(environmentID string, activityType activity.Type, message activity.Message) {
- s.publishInternal(environmentID, activity.StreamEvent{
- Type: "message",
- ActivityID: message.ActivityID,
- ActivityType: activityType,
- Message: &message,
- Timestamp: time.Now(),
- })
-}
-
-func (s *ActivityService) publishInternal(environmentID string, event activity.StreamEvent) {
- if s == nil {
- return
- }
- s.subscribersMu.RLock()
- subs := make([]*activitySubscriber, 0, len(s.subscribers))
- for _, sub := range s.subscribers {
- if sub.environmentID == environmentID {
- subs = append(subs, sub)
- }
- }
- s.subscribersMu.RUnlock()
-
- for _, sub := range subs {
- // Source labeling mutates activity fields independently for each subscriber.
- subscriberEvent := event
- if event.Activity != nil {
- subscriberEvent.Activity = new(*event.Activity)
- }
- subscriberEvent.Activities = slices.Clone(event.Activities)
- sub.enqueue(subscriberEvent)
- }
-}
-
-func activityToDTOInternal(model *Activity) activity.Activity {
+func activityToDTO(model *Activity) activity.Activity {
if model == nil {
return activity.Activity{}
}
@@ -1464,55 +1048,51 @@ func activityToDTOInternal(model *Activity) activity.Activity {
ID: model.ID,
EnvironmentID: model.EnvironmentID,
SourceEnvironmentID: model.EnvironmentID,
- BatchID: copyPtrInternal(model.BatchID),
+ BatchID: copyPtr(model.BatchID),
Type: model.Type,
Status: model.Status,
- ResourceType: copyPtrInternal(model.ResourceType),
- ResourceID: copyPtrInternal(model.ResourceID),
- ResourceName: copyPtrInternal(model.ResourceName),
- Progress: clampProgressPtrInternal(model.Progress),
+ ResourceType: copyPtr(model.ResourceType),
+ ResourceID: copyPtr(model.ResourceID),
+ ResourceName: copyPtr(model.ResourceName),
+ Progress: clampProgressPtr(model.Progress),
Step: model.Step,
LatestMessage: model.LatestMessage,
- StartedBy: activityStartedByDTOInternal(model),
- StartedAt: model.StartedAt,
- EndedAt: copyPtrInternal(model.EndedAt),
- DurationMs: copyPtrInternal(model.DurationMs),
- Error: copyPtrInternal(model.Error),
- Metadata: jsonToMapInternal(model.Metadata),
- CreatedAt: model.CreatedAt,
- UpdatedAt: copyPtrInternal(model.UpdatedAt),
- }
-}
-
-func activityMessageToDTOInternal(model *ActivityMessage) activity.Message {
- if model == nil {
- return activity.Message{}
- }
- return activity.Message{
- ID: model.ID,
- ActivityID: model.ActivityID,
- Level: model.Level,
- Message: model.Message,
- Payload: jsonToMapInternal(model.Payload),
+ StartedBy: func() *activity.StartedBy {
+ username := strings.TrimSpace(kit.FromPtr(model.StartedByUsername))
+ if username == "" {
+ return &activity.StartedBy{Username: "System"}
+ }
+ return &activity.StartedBy{
+ Username: username,
+ UserID: strings.TrimSpace(kit.FromPtr(model.StartedByUserID)),
+ DisplayName: strings.TrimSpace(kit.FromPtr(model.StartedByDisplayName)),
+ }
+ }(),
+ StartedAt: model.StartedAt,
+ EndedAt: copyPtr(model.EndedAt),
+ DurationMs: copyPtr(model.DurationMs),
+ Error: copyPtr(model.Error),
+ Metadata: cloneJSON(model.Metadata),
CreatedAt: model.CreatedAt,
+ UpdatedAt: copyPtr(model.UpdatedAt),
}
}
-func copyPtrInternal[T any](value *T) *T {
+func copyPtr[T any](value *T) *T {
if value == nil {
return nil
}
return new(*value)
}
-func clampProgressPtrInternal(value *int) *int {
+func clampProgressPtr(value *int) *int {
if value == nil {
return nil
}
return new(min(max(*value, 0), 100))
}
-func cloneJSONInternal(input database.JSON) database.JSON {
+func cloneJSON(input database.JSON) database.JSON {
if len(input) == 0 {
return nil
}
@@ -1521,54 +1101,13 @@ func cloneJSONInternal(input database.JSON) database.JSON {
return out
}
-func jsonToMapInternal(input database.JSON) map[string]any {
- if len(input) == 0 {
- return nil
- }
- out := make(map[string]any, len(input))
- maps.Copy(out, input)
- return out
-}
-
-func terminalActivityStatusesInternal() []activity.Status {
- return []activity.Status{
- activity.StatusSuccess,
- activity.StatusFailed,
- activity.StatusCancelled,
- }
-}
-
-func findTerminalActivityIDsInternal(q *gorm.DB) ([]string, error) {
- var activityIDs []string
- if err := q.Model(&Activity{}).
- Where("status IN ?", terminalActivityStatusesInternal()).
- Pluck("id", &activityIDs).Error; err != nil {
- return nil, err
- }
- return activityIDs, nil
-}
-
-func findActivityIDsBeyondHistoryLimitInternal(tx *gorm.DB, maxEntries int) ([]string, error) {
- var activityIDs []string
- if err := tx.Raw(`
- SELECT ranked.id
- FROM (
- SELECT id,
- ROW_NUMBER() OVER (
- PARTITION BY environment_id
- ORDER BY COALESCE(ended_at, updated_at, created_at) DESC, started_at DESC
- ) AS activity_rank
- FROM activities
- WHERE status IN ?
- ) ranked
- WHERE ranked.activity_rank > ?
- `, terminalActivityStatusesInternal(), maxEntries).Scan(&activityIDs).Error; err != nil {
- return nil, fmt.Errorf("failed to find excess activities: %w", err)
- }
- return activityIDs, nil
+var terminalActivityStatuses = []activity.Status{
+ activity.StatusSuccess,
+ activity.StatusFailed,
+ activity.StatusCancelled,
}
-func deleteActivitiesByIDInternal(tx *gorm.DB, activityIDs []string) (int64, error) {
+func deleteActivitiesByID(tx *gorm.DB, activityIDs []string) (int64, error) {
if len(activityIDs) == 0 {
return 0, nil
}
@@ -1591,74 +1130,38 @@ func deleteActivitiesByIDInternal(tx *gorm.DB, activityIDs []string) (int64, err
return totalDeleted, nil
}
-func activityStartedByDTOInternal(model *Activity) *activity.StartedBy {
- if model.StartedByUsername == nil || strings.TrimSpace(*model.StartedByUsername) == "" {
- return &activity.StartedBy{Username: "System"}
- }
-
- startedBy := &activity.StartedBy{
- Username: strings.TrimSpace(*model.StartedByUsername),
- }
- if model.StartedByUserID != nil {
- startedBy.UserID = strings.TrimSpace(*model.StartedByUserID)
- }
- if model.StartedByDisplayName != nil {
- startedBy.DisplayName = strings.TrimSpace(*model.StartedByDisplayName)
- }
- return startedBy
-}
-
// FailInterruptedBackups finalizes backup activities before startup admits work.
func (s *ActivityService) FailInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
- var pending []Activity
- if err := s.db.WithContext(
- ctx,
- ).Where(
- "status IN ?",
- []activity.Status{
- activity.StatusQueued,
- activity.StatusRunning,
+ _, err := s.finalizeSwept(ctx, "interrupted backup",
+ func() ([]sweeps.Candidate, error) {
+ pending, err := s.sweeps.InterruptedBackups(ctx, protectedIDs)
+ s.runningMu.Lock()
+ defer s.runningMu.Unlock()
+ return slices.DeleteFunc(pending, func(candidate sweeps.Candidate) bool {
+ _, tracked := s.running[candidate.ID]
+ return tracked
+ }), err
},
- ).Where(
- "id NOT IN ?",
- append(
- protectedIDs,
- "",
- ),
- ).Find(
- &pending,
- ).Error; err != nil {
- return fmt.Errorf("find interrupted backup activities: %w", err)
- }
- const message = "Backup interrupted by Arcane restart"
- var result error
- for _, entry := range pending {
- if s.isTrackedInternal(entry.ID) {
- continue
- }
- switch entry.Metadata["action"] {
- case "create_volume_backup", "create_system_backup", "run_system_volume_backups", "scheduled_volume_backup", "scheduled_system_backup":
- errMessage := message
- _, err := s.CompleteActivity(ctx, entry.ID, activity.StatusFailed, message, &errMessage)
- result = errors.Join(result, err)
- }
- }
- return result
+ func(sweeps.Candidate) (activity.Status, string) {
+ return activity.StatusFailed, "Backup interrupted by Arcane restart"
+ },
+ )
+ return err
}
// SyncJobRun updates a durable job summary without taking an execution slot.
// Only the queue owns its lifecycle, including reopening it for an explicit retry.
func (s *ActivityService) SyncJobRun(ctx context.Context, run scheduler.Run, name string) error {
- if err := s.checkInitInternal(); err != nil {
+ if err := s.checkInit(); err != nil {
return err
}
if run.ActivityID == "" {
return errors.New("job activity ID is required")
}
- lock := s.publishLockInternal(run.ActivityID)
+ lock := s.publishLock(run.ActivityID)
lock.Lock()
defer lock.Unlock()
- model := jobActivityInternal(run, name)
+ model := jobActivity(run, name)
changed := false
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var existing Activity
@@ -1696,15 +1199,13 @@ func (s *ActivityService) SyncJobRun(ctx context.Context, run scheduler.Run, nam
if changed {
// Ordinary activities cannot reopen. Job summaries can, after an explicit
// retry; the publish lock and durable timestamp reject stale snapshots.
- s.terminalPublishedMu.Lock()
- delete(s.terminalPublished, model.ID)
- s.terminalPublishedMu.Unlock()
- s.publishActivityInternal(activityToDTOInternal(&model))
+ s.pubsub.Reopen(model.ID)
+ s.pubsub.PublishActivity(activityToDTO(&model))
}
return nil
}
-func jobActivityInternal(run scheduler.Run, name string) Activity {
+func jobActivity(run scheduler.Run, name string) Activity {
status := activity.StatusQueued
switch run.Status {
case scheduler.Running:
@@ -1718,43 +1219,52 @@ func jobActivityInternal(run scheduler.Run, name string) Activity {
case scheduler.Queued, scheduler.Waiting, scheduler.Retrying:
}
message := cmp.Or(run.Outcome.Message, name+": "+string(run.Status))
- model := Activity{
- ID: run.ActivityID, CreatedAt: run.CreatedAt, UpdatedAt: new(run.UpdatedAt),
- EnvironmentID: run.EnvironmentID, BatchID: new(run.ID), Type: activity.TypeJobRun, Status: status,
- ResourceType: new("job"), ResourceID: new(run.JobID), ResourceName: new(name),
- StartedAt: run.CreatedAt, Step: string(run.Status), LatestMessage: message,
- Metadata: database.JSON{
- "jobId": run.JobID, "runId": run.ID, "environmentId": run.EnvironmentID,
- "jobStatus": string(run.Status), "trigger": run.Trigger, "attemptCount": run.AttemptCount,
- "runUpdatedAt": run.UpdatedAt.Format(time.RFC3339Nano), "domainActivityId": run.Outcome.ActivityID,
- },
- }
- if run.RequestedBy != "" {
- model.StartedByUserID = new(run.RequestedBy)
- model.StartedByUsername = new(run.RequestedBy)
- }
- if run.Resolution != nil {
- model.Metadata["resolution"] = run.Resolution
- if run.Resolution.ResolvedBy != user.SystemUser.Username {
- model.LatestMessage = "Resolved after review"
- }
- }
- if isTerminalActivityStatusInternal(status) {
- model.EndedAt = new(run.UpdatedAt)
- model.DurationMs = new(max(int64(0), run.UpdatedAt.Sub(run.CreatedAt).Milliseconds()))
- }
- if status == activity.StatusFailed {
- model.Error = new(message)
+ terminal := slices.Contains(terminalActivityStatuses, status)
+ resolvedByUser := run.Resolution != nil && run.Resolution.ResolvedBy != user.SystemUser.Username
+ return Activity{
+ ID: run.ActivityID,
+ CreatedAt: run.CreatedAt,
+ UpdatedAt: new(run.UpdatedAt),
+ EnvironmentID: run.EnvironmentID,
+ BatchID: new(run.ID),
+ Type: activity.TypeJobRun,
+ Status: status,
+ ResourceType: new("job"),
+ ResourceID: new(run.JobID),
+ ResourceName: new(name),
+ StartedByUserID: mo.EmptyableToOption(run.RequestedBy).ToPointer(),
+ StartedByUsername: mo.EmptyableToOption(run.RequestedBy).ToPointer(),
+ StartedAt: run.CreatedAt,
+ Step: string(run.Status),
+ LatestMessage: kit.Ternary(resolvedByUser, "Resolved after review", message),
+ EndedAt: kit.Ternary(terminal, new(run.UpdatedAt), nil),
+ DurationMs: kit.Ternary(terminal, new(max(int64(0), run.UpdatedAt.Sub(run.CreatedAt).Milliseconds())), nil),
+ Error: kit.Ternary(status == activity.StatusFailed, new(message), nil),
+ Metadata: func() database.JSON {
+ metadata := database.JSON{
+ "jobId": run.JobID,
+ "runId": run.ID,
+ "environmentId": run.EnvironmentID,
+ "jobStatus": string(run.Status),
+ "trigger": run.Trigger,
+ "attemptCount": run.AttemptCount,
+ "runUpdatedAt": run.UpdatedAt.Format(time.RFC3339Nano),
+ "domainActivityId": run.Outcome.ActivityID,
+ }
+ if run.Resolution != nil {
+ metadata["resolution"] = run.Resolution
+ }
+ return metadata
+ }(),
}
- return model
}
// remoteActivityPageCeiling mirrors the agent's page clamp in PaginateAndSortDB.
const remoteActivityPageCeiling = 100
-// normalizeRemoteActivityParamsInternal applies the agent's paging bounds so both
+// normalizeRemoteActivityParams applies the agent's paging bounds so both
// sources are cut the same way; -1 stays the explicit "show all" sentinel.
-func normalizeRemoteActivityParamsInternal(params pagination.QueryParams) pagination.QueryParams {
+func normalizeRemoteActivityParams(params pagination.QueryParams) pagination.QueryParams {
params.Start = max(params.Start, 0)
switch {
case params.Limit == -1:
@@ -1766,9 +1276,9 @@ func normalizeRemoteActivityParamsInternal(params pagination.QueryParams) pagina
return params
}
-// collectActivityWindowInternal reads window rows from a source in pages the
+// collectActivityWindow reads window rows from a source in pages the
// agent accepts, stopping early when the source runs out. -1 reads everything.
-func collectActivityWindowInternal(window int, fetchPage func(start, limit int) ([]activity.Activity, int64, error)) ([]activity.Activity, int64, error) {
+func collectActivityWindow(window int, fetchPage func(start, limit int) ([]activity.Activity, int64, error)) ([]activity.Activity, int64, error) {
if window == -1 {
return fetchPage(0, -1)
}
@@ -1792,22 +1302,16 @@ func collectActivityWindowInternal(window int, fetchPage func(start, limit int)
// ListRemoteActivities combines manager-owned summaries with agent-owned work
// before slicing the page. The remote records must already match the filters
// and be the agent's newest window; remoteTotal is the agent's filtered count.
-func (
- s *ActivityService,
-) ListRemoteActivities(
+func (s *ActivityService) ListRemoteActivities(
ctx context.Context,
environmentID string,
remote []activity.Activity,
remoteTotal int64,
params pagination.QueryParams,
-) (
- []activity.Activity,
- pagination.Response,
- error,
-) {
- params = normalizeRemoteActivityParamsInternal(params)
+) ([]activity.Activity, pagination.Response, error) {
+ params = normalizeRemoteActivityParams(params)
window := kit.Ternary(params.Limit == -1, -1, params.Start+params.Limit)
- activities, localTotal, err := collectActivityWindowInternal(window, func(start, limit int) ([]activity.Activity, int64, error) {
+ activities, localTotal, err := collectActivityWindow(window, func(start, limit int) ([]activity.Activity, int64, error) {
local := params
local.Start, local.Limit = start, limit
rows, page, err := s.ListActivitiesPaginated(ctx, environmentID, local)
@@ -1825,7 +1329,7 @@ func (
}
total := remoteTotal + localTotal
config := pagination.Config[activity.Activity]{SortBindings: []pagination.SortBinding[activity.Activity]{{
- Fn: func(a, b activity.Activity) int { return compareActivitiesInternal(a, b, params) },
+ Fn: func(a, b activity.Activity) int { return compareActivities(a, b, params) },
}}}
pageParams := params
pageParams.Order = pagination.SortAsc
@@ -1833,7 +1337,7 @@ func (
return activities, pagination.BuildResponse(total, 0, params), nil
}
-func compareActivitiesInternal(a, b activity.Activity, params pagination.QueryParams) int {
+func compareActivities(a, b activity.Activity, params pagination.QueryParams) int {
if params.Sort == "" {
aActive := a.Status == activity.StatusQueued || a.Status == activity.StatusRunning
bActive := b.Status == activity.StatusQueued || b.Status == activity.StatusRunning
diff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.go
index 3fa42d4e3..3c0a44e9f 100644
--- a/backend/internal/activity/service_test.go
+++ b/backend/internal/activity/service_test.go
@@ -31,7 +31,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
-func setupActivityServiceTestDBInternal(t *testing.T) *database.DB {
+func setupActivityServiceTestDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -40,9 +40,9 @@ func setupActivityServiceTestDBInternal(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func TestActivityServiceLifecycleInternal(t *testing.T) {
+func TestActivityServiceLifecycle(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
progress := 5
@@ -106,14 +106,14 @@ func TestActivityServiceLifecycleInternal(t *testing.T) {
require.Equal(t, "Pull complete", detail.Messages[1].Message)
}
-func TestActivityServiceStreamFanoutInternal(t *testing.T) {
+func TestActivityServiceStreamFanout(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
- events, _, unsubscribe := service.Subscribe("0")
+ events, _, unsubscribe := service.pubsub.Subscribe("0")
defer unsubscribe()
- otherEvents, _, unsubscribeOther := service.Subscribe("0")
+ otherEvents, _, unsubscribeOther := service.pubsub.Subscribe("0")
defer unsubscribeOther()
created, err := service.StartActivity(ctx, StartActivityRequest{
@@ -123,11 +123,11 @@ func TestActivityServiceStreamFanoutInternal(t *testing.T) {
})
require.NoError(t, err)
- first := receiveActivityEventInternal(t, events)
+ first := receiveActivityEvent(t, events)
require.Equal(t, "activity", first.Type)
require.Equal(t, created.ID, first.ActivityID)
require.NotNil(t, first.Activity)
- other := receiveActivityEventInternal(t, otherEvents)
+ other := receiveActivityEvent(t, otherEvents)
require.Equal(t, first, other)
require.NotSame(t, first.Activity, other.Activity)
@@ -138,7 +138,7 @@ func TestActivityServiceStreamFanoutInternal(t *testing.T) {
})
require.NoError(t, err)
- messageEvent := receiveActivityEventInternal(t, events)
+ messageEvent := receiveActivityEvent(t, events)
require.Equal(t, "message", messageEvent.Type)
require.Equal(t, created.ID, messageEvent.ActivityID)
require.Equal(t, activity.TypeProjectDeploy, messageEvent.ActivityType)
@@ -146,12 +146,12 @@ func TestActivityServiceStreamFanoutInternal(t *testing.T) {
require.Equal(t, "Deploying services", messageEvent.Message.Message)
require.Nil(t, messageEvent.Activity)
require.Nil(t, messageEvent.Activities)
- require.Equal(t, messageEvent, receiveActivityEventInternal(t, otherEvents))
+ require.Equal(t, messageEvent, receiveActivityEvent(t, otherEvents))
}
-func TestActivityServiceRetentionCleanupInternal(t *testing.T) {
+func TestActivityServiceRetentionCleanup(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
created, err := service.StartActivity(ctx, StartActivityRequest{
@@ -183,9 +183,9 @@ func TestActivityServiceRetentionCleanupInternal(t *testing.T) {
require.Zero(t, messageCount)
}
-func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanupInternal(t *testing.T) {
+func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanup(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
created, err := service.StartActivity(ctx, StartActivityRequest{
@@ -209,26 +209,9 @@ func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanupInternal(t *t
require.EqualValues(t, 1, activityCount)
}
-func TestActivityServiceSubscribeMarksMissedEventsWhenBufferFullInternal(t *testing.T) {
- service := NewActivityService(nil, nil)
-
- events, missedEvents, unsubscribe := service.Subscribe("0")
- defer unsubscribe()
-
- // Message-type events are not coalesced; overflowing the delivery channel
- // plus the bounded pending FIFO must flag missed events so the stream
- // handler resends a snapshot.
- total := cap(events) + subscriberMessageQueueLimit + 50
- for range total {
- service.publishInternal("0", activity.StreamEvent{Type: "message"})
- }
- require.True(t, missedEvents())
- require.False(t, missedEvents())
-}
-
-func TestActivityServiceDeleteHistoryPreservesActiveActivitiesInternal(t *testing.T) {
+func TestActivityServiceDeleteHistoryPreservesActiveActivities(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
completed, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})
@@ -256,9 +239,9 @@ func TestActivityServiceDeleteHistoryPreservesActiveActivitiesInternal(t *testin
require.ElementsMatch(t, []string{running.ID, remoteCompleted.ID}, []string{remaining[0].ID, remaining[1].ID})
}
-func TestActivityServicePruneHistoryByAgeAndCountInternal(t *testing.T) {
+func TestActivityServicePruneHistoryByAgeAndCount(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
oldActivity, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})
@@ -292,7 +275,7 @@ func TestActivityServicePruneHistoryByAgeAndCountInternal(t *testing.T) {
var terminalRemoteCount int64
require.NoError(t, db.Model(&Activity{}).
- Where("environment_id = ? AND status IN ?", "remote-1", terminalActivityStatusesInternal()).
+ Where("environment_id = ? AND status IN ?", "remote-1", terminalActivityStatuses).
Count(&terminalRemoteCount).Error)
require.EqualValues(t, 2, terminalRemoteCount)
@@ -305,47 +288,9 @@ func TestActivityServicePruneHistoryByAgeAndCountInternal(t *testing.T) {
require.Zero(t, oldCount)
}
-func TestActivitySubscriberCoalescesProgressEventsInternal(t *testing.T) {
- db := setupActivityServiceTestDBInternal(t)
- service := NewActivityService(db, nil)
-
- events, missedEvents, unsubscribe := service.Subscribe("0")
- defer unsubscribe()
-
- const updates = 500
- for i := 1; i <= updates; i++ {
- progress := i * 100 / updates
- service.publishActivityInternal(activity.Activity{
- ID: "act-1",
- EnvironmentID: "0",
- Status: activity.StatusRunning,
- Progress: &progress,
- })
- }
-
- received := 0
- deadline := time.After(5 * time.Second)
- for {
- select {
- case event := <-events:
- received++
- require.Equal(t, "activity", event.Type)
- if event.Activity != nil && event.Activity.Progress != nil && *event.Activity.Progress == 100 {
- // The consumer was idle during publishing, so the backlog must
- // have been coalesced instead of delivered event-for-event.
- require.Less(t, received, updates)
- require.False(t, missedEvents())
- return
- }
- case <-deadline:
- require.FailNow(t, "did not receive final coalesced progress event")
- }
- }
-}
-
-func TestActivityServiceListOrderStableUnderProgressUpdatesInternal(t *testing.T) {
+func TestActivityServiceListOrderStableUnderProgressUpdates(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
older, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull})
@@ -381,7 +326,7 @@ func TestActivityServiceListOrderStableUnderProgressUpdatesInternal(t *testing.T
require.Equal(t, expected, listIDs())
}
-func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, context.Context) {
+func setupQueuedActivityService(t *testing.T) (*ActivityService, *settings.SettingsService, context.Context) {
t.Helper()
ctx := t.Context()
@@ -396,13 +341,13 @@ func setupQueuedActivityServiceInternal(t *testing.T) (*ActivityService, context
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
- localSettings, err := newSettingsServiceForTestInternal(t, ctx, wrapped)
+ localSettings, err := newSettingsServiceForTest(t, ctx, wrapped)
require.NoError(t, err)
- require.NoError(t, localSettings.SetIntSetting(ctx, maxConcurrentActivitiesSettingKey, 1))
- return NewActivityService(wrapped, localSettings), ctx
+ require.NoError(t, localSettings.SetIntSetting(ctx, "maxConcurrentActivities", 1))
+ return NewActivityService(wrapped, localSettings), localSettings, ctx
}
-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
svc, err := settings.NewSettingsService(ctx, db)
if err == nil {
@@ -411,8 +356,8 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *da
return svc, err
}
-func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFreesInternal(t *testing.T) {
- service, ctx := setupQueuedActivityServiceInternal(t)
+func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFrees(t *testing.T) {
+ service, _, ctx := setupQueuedActivityService(t)
first, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
require.NoError(t, err)
@@ -459,8 +404,8 @@ func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFreesInternal(t *tes
require.NoError(t, service.AwaitActivitySlot(ctx, secondDeferred.ID, "0"))
}
-func TestActivityServiceLimitIncreaseKeepsCountingActiveSlotsInternal(t *testing.T) {
- service, ctx := setupQueuedActivityServiceInternal(t)
+func TestActivityServiceLimitIncreaseKeepsCountingActiveSlots(t *testing.T) {
+ service, localSettings, ctx := setupQueuedActivityService(t)
running, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
require.NoError(t, err)
@@ -474,12 +419,12 @@ func TestActivityServiceLimitIncreaseKeepsCountingActiveSlotsInternal(t *testing
// Raising the limit to 2 must admit the waiter while still counting the
// original holder, leaving no capacity for a third activity.
- require.NoError(t, service.limiter.settings.SetIntSetting(ctx, maxConcurrentActivitiesSettingKey, 2))
+ require.NoError(t, localSettings.SetIntSetting(ctx, "maxConcurrentActivities", 2))
select {
case awaitErr := <-awaitDone:
require.NoError(t, awaitErr)
- case <-time.After(2 * slotWaitRecheckInterval):
+ case <-time.After(4 * time.Second):
require.FailNow(t, "waiter was not admitted after the limit increase")
}
@@ -488,8 +433,8 @@ func TestActivityServiceLimitIncreaseKeepsCountingActiveSlotsInternal(t *testing
require.Equal(t, "queued", string(third.Status))
}
-func TestActivityServiceCancelWhileQueuedUnblocksAwaitInternal(t *testing.T) {
- service, ctx := setupQueuedActivityServiceInternal(t)
+func TestActivityServiceCancelWhileQueuedUnblocksAwait(t *testing.T) {
+ service, _, ctx := setupQueuedActivityService(t)
_, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
require.NoError(t, err)
@@ -511,14 +456,14 @@ func TestActivityServiceCancelWhileQueuedUnblocksAwaitInternal(t *testing.T) {
}
}
-func TestActivityServiceCompleteActivityRejectsUninitializedServiceInternal(t *testing.T) {
+func TestActivityServiceCompleteActivityRejectsUninitializedService(t *testing.T) {
service := NewActivityService(nil, nil)
_, err := service.CompleteActivity(t.Context(), "any-id", activity.StatusSuccess, "done", nil)
require.Error(t, err)
}
-func TestActivityServiceTrackAndRequestCancelInternal(t *testing.T) {
- db := setupActivityServiceTestDBInternal(t)
+func TestActivityServiceTrackAndRequestCancel(t *testing.T) {
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
// Mirror the handler flow: work runs under an app-lifecycle runtime context.
@@ -552,9 +497,9 @@ func TestActivityServiceTrackAndRequestCancelInternal(t *testing.T) {
require.False(t, service.RequestCancel(created.ID))
}
-func TestActivityServiceCancelActivityInternal(t *testing.T) {
+func TestActivityServiceCancelActivity(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
// An untracked running activity (e.g. after a restart) is finalized directly.
@@ -579,9 +524,9 @@ func TestActivityServiceCancelActivityInternal(t *testing.T) {
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
}
-func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {
+func TestActivityServiceFailStaleImageUpdateChecks(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
staleCheck, err := service.StartActivity(ctx, StartActivityRequest{
@@ -647,9 +592,9 @@ func TestActivityServiceFailStaleImageUpdateChecksInternal(t *testing.T) {
require.Equal(t, activity.StatusSuccess, completed.Status)
}
-func TestActivityServiceFailAbandonedActivitiesInternal(t *testing.T) {
+func TestActivityServiceFailAbandonedActivities(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
abandoned, err := service.StartActivity(ctx, StartActivityRequest{
@@ -696,9 +641,9 @@ func TestActivityServiceFailAbandonedActivitiesInternal(t *testing.T) {
require.Equal(t, activity.StatusRunning, freshRow.Status)
}
-func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {
+func TestActivityServiceResolveStaleAutoUpdateActivities(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
selfUpdateRun, err := service.StartActivity(ctx, StartActivityRequest{
@@ -744,7 +689,7 @@ func TestActivityServiceResolveStaleAutoUpdateActivitiesInternal(t *testing.T) {
require.Equal(t, activity.StatusRunning, other.Status)
}
-func receiveActivityEventInternal(t *testing.T, events <-chan activity.StreamEvent) activity.StreamEvent {
+func receiveActivityEvent(t *testing.T, events <-chan activity.StreamEvent) activity.StreamEvent {
t.Helper()
select {
@@ -756,12 +701,12 @@ func receiveActivityEventInternal(t *testing.T, events <-chan activity.StreamEve
}
}
-// TestActivityServiceAppendMessagesBatchInternal verifies a batch lands all
+// TestActivityServiceAppendMessagesBatch verifies a batch lands all
// messages in order and coalesces the activity update to the batch's last
// message, last progress, and last step.
-func TestActivityServiceAppendMessagesBatchInternal(t *testing.T) {
+func TestActivityServiceAppendMessagesBatch(t *testing.T) {
ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
created, err := service.StartActivity(ctx, StartActivityRequest{
@@ -803,48 +748,8 @@ func TestActivityServiceAppendMessagesBatchInternal(t *testing.T) {
require.ErrorContains(t, err, "activity not found")
}
-// TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal verifies
-// that a non-terminal activity snapshot published after the activity's
-// terminal event — a goroutine that committed before CompleteActivity but
-// publishes after it — is dropped instead of reverting subscribers to a
-// running state no later event would correct.
-func TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal(t *testing.T) {
- ctx := t.Context()
- db := setupActivityServiceTestDBInternal(t)
- service := NewActivityService(db, nil)
-
- created, err := service.StartActivity(ctx, StartActivityRequest{
- EnvironmentID: "0",
- Type: activity.TypeImagePull,
- LatestMessage: "Pull queued",
- })
- require.NoError(t, err)
- stale := *created // snapshot taken while the activity is still active
-
- events, _, unsubscribe := service.Subscribe("0")
- defer unsubscribe()
-
- completed, err := service.CompleteActivity(ctx, created.ID, activity.StatusSuccess, "", nil)
- require.NoError(t, err)
- require.Equal(t, activity.StatusSuccess, completed.Status)
-
- event := receiveActivityEventInternal(t, events)
- require.Equal(t, "activity", event.Type)
- require.Equal(t, activity.StatusSuccess, event.Activity.Status)
-
- require.False(t, service.admitActivityPublishInternal(stale))
- require.True(t, service.admitActivityPublishInternal(*completed))
-
- service.publishActivityInternal(stale)
- select {
- case localEvent := <-events:
- t.Fatalf("stale non-terminal snapshot reached subscriber: %+v", localEvent)
- case <-time.After(100 * time.Millisecond):
- }
-}
-
func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
now := time.Now().UTC()
run := scheduler.Run{ID: "run", ActivityID: "summary", JobID: "auto-update", EnvironmentID: "0", Status: scheduler.NeedsAttention, CreatedAt: now.Add(-time.Hour), UpdatedAt: now}
@@ -859,9 +764,7 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {
require.Equal(t, activity.StatusQueued, detail.Activity.Status)
require.Nil(t, detail.Activity.EndedAt)
require.Nil(t, detail.Activity.Error)
- require.Empty(t, service.slotReleases)
require.Empty(t, service.running)
- require.NotContains(t, service.terminalPublished, run.ActivityID)
count, err := service.ResolveOrphanedQueuedActivities(t.Context())
require.NoError(t, err)
require.Zero(t, count)
@@ -878,7 +781,7 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {
require.Equal(t, "remote", legacy.EnvironmentID)
// Startup reconciliation also repairs legacy rows without replaying the job.
require.NoError(t, db.Model(&legacy).Update("environment_id", "0").Error)
- events, _, unsubscribe := service.Subscribe("remote")
+ events, _, unsubscribe := service.pubsub.Subscribe("remote")
defer unsubscribe()
require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))
select {
@@ -919,7 +822,7 @@ func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {
}
func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
for _, environmentID := range []string{"0", "private"} {
require.NoError(
@@ -954,7 +857,7 @@ func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, 2, response.TotalItems)
require.Len(t, activities, 1)
- require.True(t, canReadJobActivityInternal(ctx, activities[0]))
+ require.True(t, canReadJobActivity(ctx, activities[0]))
seen[activities[0].ID] = true
}
require.Len(t, seen, 2)
@@ -965,7 +868,7 @@ func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {
require.Equal(t, "private-pull", activities[0].ID)
require.False(
t,
- canReadJobActivityInternal(
+ canReadJobActivity(
ctx,
activity.Activity{
Type: activity.TypeJobRun,
@@ -974,13 +877,13 @@ func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {
},
),
)
- require.True(t, canReadJobActivityInternal(ctx, activity.Activity{Type: activity.TypeJobRun, Metadata: map[string]any{"environmentId": "allowed"}}))
+ require.True(t, canReadJobActivity(ctx, activity.Activity{Type: activity.TypeJobRun, Metadata: map[string]any{"environmentId": "allowed"}}))
privileged := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.SudoPermissionSet())
activities, response, err = service.ListActivitiesPaginated(privileged, "private", pagination.QueryParams{Limit: 10})
require.NoError(t, err)
require.Len(t, activities, 2)
require.EqualValues(t, 2, response.TotalItems)
- require.False(t, canReadJobActivityInternal(privileged, activity.Activity{Type: activity.TypeJobRun}))
+ require.False(t, canReadJobActivity(privileged, activity.Activity{Type: activity.TypeJobRun}))
permissions.PerEnv["0"][authz.PermActivitiesDelete] = struct{}{}
deleted, err := service.DeleteHistory(ctx, "private")
require.NoError(t, err)
@@ -1009,7 +912,7 @@ func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
require.NoError(t, db.AutoMigrate(&environment.Environment{}, &registry.ContainerRegistry{}, &gitrepo.GitRepository{}, &s3.S3Destination{}))
sqlDB, err := db.DB.DB()
require.NoError(t, err)
@@ -1066,7 +969,7 @@ func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {
}
func TestActivityService_StartActivityWithIDIsIdempotent(t *testing.T) {
- db := setupActivityServiceTestDBInternal(t)
+ db := setupActivityServiceTestDB(t)
service := NewActivityService(db, nil)
first, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "first"})
diff --git a/backend/internal/backup/service.go b/backend/internal/backup/service.go
index 9fd7d45f3..830492928 100644
--- a/backend/internal/backup/service.go
+++ b/backend/internal/backup/service.go
@@ -520,6 +520,10 @@ func (e *Engine) ForgetSnapshots(ctx context.Context, dockerClient *client.Clien
command = append(command, listedID)
}
}
+ if len(command) == 2 && len(requested) > 0 {
+ slog.DebugContext(ctx, "No requested snapshots found in repository; skipping forget", "repositoryId", repository.ID, "requested", requested)
+ return nil
+ }
if len(command) == 2 {
command = []string{"prune"}
}
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index b2f7e2b7e..e91cb85e4 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -16,6 +16,7 @@ import (
"github.com/labstack/echo/v5"
"github.com/moby/moby/client"
"github.com/subosito/gotenv"
+ kit "go.getarcane.app/kit/pkg"
"go.getarcane.app/sys/crypto"
"go.opentelemetry.io/contrib/bridges/otelslog"
"go.uber.org/fx"
@@ -23,6 +24,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/di"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -144,7 +146,12 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
fx.Supply(cfg, db, cancelApp),
fx.Provide(
func() context.Context { return appCtx },
- newConfiguredHTTPClient,
+ func(cfg *config.Config) *http.Client {
+ return httpx.NewHTTPClient(httpxtypes.ClientOptions{
+ Timeout: kit.Ternary(cfg.HTTPClientTimeout > 0, time.Duration(cfg.HTTPClientTimeout)*time.Second, 10*time.Second),
+ TLSHandshakeTimeout: kit.Ternary(cfg.HTTPClientTimeout > 0, 10*time.Second, 5*time.Second),
+ })
+ },
),
di.ActorOptions,
di.ServiceOptions,
@@ -162,7 +169,7 @@ func applicationOptions(appCtx context.Context, cfg *config.Config, db *database
return logger
}),
fx.StartTimeout(5*time.Minute),
- fx.StopTimeout(30*time.Second),
+ fx.StopTimeout(8*time.Second),
)
}
@@ -179,18 +186,6 @@ func isWeakProductionEncryptionKey(encryptionKey, localEnvironment string, agent
return len(strings.TrimPrefix(key, "raw:")) < 32
}
-func newConfiguredHTTPClient(cfg *config.Config) *http.Client {
- options := httpxtypes.ClientOptions{
- Timeout: 10 * time.Second,
- TLSHandshakeTimeout: 5 * time.Second,
- }
- if cfg.HTTPClientTimeout > 0 {
- options.Timeout = time.Duration(cfg.HTTPClientTimeout) * time.Second
- options.TLSHandshakeTimeout = 10 * time.Second
- }
- return httpx.NewHTTPClient(options)
-}
-
type initializeStartupStateParams struct {
fx.In
@@ -203,6 +198,7 @@ type initializeStartupStateParams struct {
Environment *environment.EnvironmentService
GitOpsSync *gitops.GitOpsSyncService
Project *project.ProjectService
+ Container *container.ContainerService
Variable *variable.VariableService
Docker *docker.DockerClientService
Swarm *swarm.SwarmService
@@ -247,34 +243,30 @@ func initializeStartupState(p initializeStartupStateParams) {
})
startup.InitializeDefaultSettings(appCtx, runtimeCfg, p.Settings)
- if err := p.Settings.NormalizeProjectsDirectory(appCtx, cfg.ProjectsDirectory); err != nil {
- slog.WarnContext(appCtx, "Failed to normalize projects directory", "error", err)
- }
-
- if err := p.Settings.NormalizeBuildsDirectory(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to normalize builds directory", "error", err)
- }
-
- if err := p.Environment.EnsureLocalEnvironment(appCtx, cfg.AppUrl); err != nil {
- slog.WarnContext(appCtx, "Failed to ensure local environment", "error", err)
- }
- if err := p.GitOpsSync.CleanupOrphanedSyncsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up orphaned GitOps syncs on startup", "error", err)
- }
- if err := p.GitOpsSync.CleanupLeakedScratchDirsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up leaked GitOps scratch directories on startup", "error", err)
- }
- if err := p.GitOpsSync.CleanupLeakedCloneDirsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to clean up leaked git clone directories on startup", "error", err)
- }
- if err := p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to reconcile directory GitOps projects on startup", "error", err)
+ // Startup repairs are best-effort: log each failure and continue.
+ for _, task := range []struct {
+ name string
+ run func(context.Context) error
+ }{
+ {name: "normalize projects directory", run: func(ctx context.Context) error {
+ return p.Settings.NormalizeProjectsDirectory(ctx, cfg.ProjectsDirectory)
+ }},
+ {name: "normalize builds directory", run: p.Settings.NormalizeBuildsDirectory},
+ {name: "ensure local environment", run: func(ctx context.Context) error { return p.Environment.EnsureLocalEnvironment(ctx, cfg.AppUrl) }},
+ {name: "clean up orphaned GitOps syncs", run: p.GitOpsSync.CleanupOrphanedSyncsOnStartup},
+ {name: "clean up leaked GitOps scratch directories", run: p.GitOpsSync.CleanupLeakedScratchDirsOnStartup},
+ {name: "clean up leaked git clone directories", run: p.GitOpsSync.CleanupLeakedCloneDirsOnStartup},
+ {name: "reconcile directory GitOps projects", run: p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup},
+ {name: "recover interrupted project renames", run: p.Project.RecoverProjectRenameJournals},
+ {name: "recover interrupted container redeploys", run: p.Container.RecoverInterruptedRedeploys},
+ {name: "recover stuck project statuses", run: p.Project.RecoverStuckProjectStatuses},
+ } {
+ if err := task.run(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Startup task failed", "task", task.name, "error", err)
+ }
}
p.Vulnerability.ImportLegacyReportFiles(appCtx)
p.Vulnerability.BackfillScanItems(appCtx)
- if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {
- slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)
- }
if !cfg.AgentMode {
if err := p.Environment.ReconcileEdgeStatusesOnStartup(appCtx); err != nil {
@@ -291,19 +283,17 @@ func initializeStartupState(p initializeStartupStateParams) {
}
startup.TestDockerConnection(appCtx, func(ctx context.Context) error {
- dockerClient, err := p.Docker.GetClient(ctx)
- if err != nil {
- return err
- }
-
- version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})
- if err != nil {
- return err
- }
+ _, err := p.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (struct{}, error) {
+ version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})
+ if err != nil {
+ return struct{}{}, err
+ }
- effectiveAPIVersion := cmp.Or(strings.TrimSpace(dockerClient.ClientVersion()), strings.TrimSpace(version.APIVersion))
- slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)
- return nil
+ effectiveAPIVersion := cmp.Or(strings.TrimSpace(dockerClient.ClientVersion()), strings.TrimSpace(version.APIVersion))
+ slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)
+ return struct{}{}, nil
+ })
+ return err
})
if err := p.Swarm.SyncSwarmEnabledState(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)
@@ -313,7 +303,7 @@ func initializeStartupState(p initializeStartupStateParams) {
p.Role.EnsureBuiltInRoles,
func(ctx context.Context) error {
// Backfill legacy users.roles first so CreateDefaultAdmin's zero-global-admin gate sees them;
- // runRoleStartupTasks repeats it for agent mode and retries.
+ // the role startup tasks below repeat it for agent mode and retries.
if err := p.Role.BackfillLegacyRoleAssignments(ctx); err != nil {
slog.WarnContext(ctx, "Failed to backfill legacy role assignments before admin bootstrap", "error", err)
}
@@ -329,7 +319,24 @@ func initializeStartupState(p initializeStartupStateParams) {
)
startup.CleanupUnknownSettings(appCtx, p.Settings)
- runRoleStartupTasks(appCtx, p.DB, p.Role, p.ApiKey, cfg)
+ if err := p.Role.EnsureBuiltInRoles(appCtx); err != nil {
+ slog.ErrorContext(appCtx, "Failed to reconcile built-in roles", "error", err)
+ }
+ // Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.
+ if err := p.Role.BackfillLegacyRoleAssignments(appCtx); err != nil {
+ slog.ErrorContext(appCtx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)
+ }
+ if err := p.ApiKey.BackfillApiKeyPermissions(appCtx); err != nil {
+ slog.WarnContext(appCtx, "Failed to backfill API key permissions", "error", err)
+ }
+ if err := p.Role.ReconcileEnvOidcMappings(appCtx, cfg.OidcRoleMappings); err != nil {
+ slog.ErrorContext(appCtx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)
+ }
+ if !cfg.AgentMode {
+ if err := p.Role.AssertGlobalAdminExists(appCtx, p.DB.WithContext(appCtx)); err != nil {
+ slog.ErrorContext(appCtx, "RBAC global admin guard failed", "error", err)
+ }
+ }
// Only edge agents auto-pair; direct agents are passive and the manager dials them.
if cfg.AgentMode && cfg.EdgeAgent && cfg.AgentToken != "" && cfg.ManagerApiUrl != "" {
@@ -341,33 +348,6 @@ func initializeStartupState(p initializeStartupStateParams) {
}
}
-func runRoleStartupTasks(ctx context.Context, db *database.DB, roleService *role.RoleService, apiKeyService *apikey.ApiKeyService, cfg *config.Config) {
- if roleService == nil {
- return
- }
- if err := roleService.EnsureBuiltInRoles(ctx); err != nil {
- slog.ErrorContext(ctx, "Failed to reconcile built-in roles", "error", err)
- }
- // Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.
- if err := roleService.BackfillLegacyRoleAssignments(ctx); err != nil {
- slog.ErrorContext(ctx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)
- }
- if apiKeyService != nil {
- if err := apiKeyService.BackfillApiKeyPermissions(ctx); err != nil {
- slog.WarnContext(ctx, "Failed to backfill API key permissions", "error", err)
- }
- }
- if err := roleService.ReconcileEnvOidcMappings(ctx, cfg.OidcRoleMappings); err != nil {
- slog.ErrorContext(ctx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)
- }
- if cfg.AgentMode {
- return
- }
- if err := roleService.AssertGlobalAdminExists(ctx, db.WithContext(ctx)); err != nil {
- slog.ErrorContext(ctx, "RBAC global admin guard failed", "error", err)
- }
-}
-
func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, router *echo.Echo, _ *http.Server) {
var stop func(context.Context) error
lc.Append(fx.Hook{
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.go
index 48e3257d9..ad820fd05 100644
--- a/backend/internal/bootstrap/bootstrap_test.go
+++ b/backend/internal/bootstrap/bootstrap_test.go
@@ -28,6 +28,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/health"
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
@@ -200,7 +201,8 @@ func TestHTTP2APIResponsesDoNotUseAPIGzip(t *testing.T) {
Project: project.New(&project.ProjectService{}, nil),
Container: container.New(&container.ContainerService{}, nil, nil, nil),
Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),
- System: system.New(&system.SystemService{}, nil, nil, nil),
+ System: system.New(&system.SystemService{}, nil, nil),
+ Health: health.New(&health.Service{}),
},
AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),
TunnelRegistry: edge.NewTunnelRegistry(),
diff --git a/backend/internal/bootstrap/router_bootstrap.go b/backend/internal/bootstrap/router_bootstrap.go
index 993dda67a..e214c21d3 100644
--- a/backend/internal/bootstrap/router_bootstrap.go
+++ b/backend/internal/bootstrap/router_bootstrap.go
@@ -295,7 +295,7 @@ func newRouter(p RouterParams) (*echo.Echo, *edge.TunnelServer) {
}
// Remaining echo handlers (WebSocket/streaming)
- ws.NewWebSocketHandler(apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)
+ ws.NewWebSocketHandler(ctx, apiGroup, deps.Project.Service(), deps.Container.Service(), deps.Swarm.Service(), deps.System.Service(), deps.Diagnostics, authMiddleware, cfg)
// Managers accept agent connections on the edge tunnel endpoint.
var tunnelServer *edge.TunnelServer
diff --git a/backend/internal/bootstrap/router_bootstrap_test.go b/backend/internal/bootstrap/router_bootstrap_test.go
index 3893f99ec..d8e0049cf 100644
--- a/backend/internal/bootstrap/router_bootstrap_test.go
+++ b/backend/internal/bootstrap/router_bootstrap_test.go
@@ -19,6 +19,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/health"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
@@ -40,7 +41,8 @@ func TestNewRouter_DecodesPathParams(t *testing.T) {
Project: project.New(&project.ProjectService{}, nil),
Container: container.New(&container.ContainerService{}, nil, nil, nil),
Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),
- System: system.New(&system.SystemService{}, nil, nil, nil),
+ System: system.New(&system.SystemService{}, nil, nil),
+ Health: health.New(&health.Service{}),
},
AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),
TunnelRegistry: edge.NewTunnelRegistry(),
@@ -156,7 +158,8 @@ func TestProxiedEnvironmentRoutesAreEdgeRoutable(t *testing.T) {
Project: project.New(&project.ProjectService{}, nil),
Container: container.New(&container.ContainerService{}, nil, nil, nil),
Swarm: swarm.New(&swarm.SwarmService{}, nil, nil, nil),
- System: system.New(&system.SystemService{}, nil, nil, nil),
+ System: system.New(&system.SystemService{}, nil, nil),
+ Health: health.New(&health.Service{}),
},
AuthMiddleware: auth.NewAuthMiddleware(nil, cfg),
TunnelRegistry: edge.NewTunnelRegistry(),
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
index d487ac683..3810e4a41 100644
--- a/backend/internal/config/config.go
+++ b/backend/internal/config/config.go
@@ -35,7 +35,7 @@ type Config struct {
BuildablesConfig
AppUrl string `env:"APP_URL" default:"http://localhost:3552"`
- DatabaseURL string `env:"DATABASE_URL" default:"file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate" options:"file"`
+ DatabaseURL string `env:"DATABASE_URL" default:"file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate" options:"file"`
AllowDowngrade bool `env:"ALLOW_DOWNGRADE" default:"false"`
AllowCLIPasswordReset bool `env:"ALLOW_CLI_PASSWORD_RESET" default:"false"`
AllowCLIMFAReset bool `env:"ALLOW_CLI_MFA_RESET" default:"false"`
@@ -118,7 +118,7 @@ func Load() *Config {
cfg := &Config{}
loadFromEnv(cfg)
applyOptions(cfg)
- applyAgentModeDefaults(cfg)
+ cfg.AgentMode = cfg.AgentMode || cfg.EdgeAgent
applyProxyDefaults(cfg)
applyWorkspaceDefaults(cfg)
@@ -172,12 +172,6 @@ func applyWorkspaceDefaults(cfg *Config) {
}
}
-func applyAgentModeDefaults(cfg *Config) {
- if cfg.EdgeAgent {
- cfg.AgentMode = true
- }
-}
-
// applyProxyDefaults auto-trusts loopback proxies when LISTEN binds to a
// loopback address. This is the canonical convenience-script setup
// (LISTEN=127.0.0.1 + local reverse proxy like Caddy/Nginx/Traefik); auto-
@@ -216,7 +210,7 @@ func loadFromEnv(cfg *Config) {
envValue := os.Getenv(envTag)
var fromFile bool
- if hasOptionInternal(fieldType, "file") {
+ if hasOption(fieldType, "file") {
envValue, _, fromFile = utils.LookupEnvOrFile(envTag)
}
if !fromFile {
@@ -226,12 +220,12 @@ func loadFromEnv(cfg *Config) {
}
}
- setFieldValueInternal(field, fieldType, envValue)
+ setFieldValue(field, fieldType, envValue)
})
}
-// hasOptionInternal reports whether the field's `options` tag lists option.
-func hasOptionInternal(fieldType reflect.StructField, option string) bool {
+// hasOption reports whether the field's `options` tag lists option.
+func hasOption(fieldType reflect.StructField, option string) bool {
return slices.Contains(strings.Split(fieldType.Tag.Get("options"), ","), option)
}
@@ -241,7 +235,7 @@ func (c *Config) DeprecatedEnvVarsSet() []string {
var envNames []string
v := reflect.ValueOf(c).Elem()
visitConfigFields(v, func(field reflect.Value, fieldType reflect.StructField) {
- if !hasOptionInternal(fieldType, "deprecated") {
+ if !hasOption(fieldType, "deprecated") {
return
}
if field.Kind() == reflect.String && field.String() == "" {
@@ -315,8 +309,8 @@ func visitConfigFields(v reflect.Value, fn func(reflect.Value, reflect.StructFie
}
}
-// setFieldValueInternal sets a reflect.Value from a string based on the field's type.
-func setFieldValueInternal(field reflect.Value, fieldType reflect.StructField, value string) {
+// setFieldValue sets a reflect.Value from a string based on the field's type.
+func setFieldValue(field reflect.Value, fieldType reflect.StructField, value string) {
ctx := context.Background() //nolint:forbidigo // Configuration loads before any request context exists.
if !field.CanSet() {
return
@@ -350,47 +344,26 @@ func setFieldValueInternal(field reflect.Value, fieldType reflect.StructField, v
}
if field.Type() == reflect.TypeFor[time.Duration]() {
- applyDurationDefault := func(reason string) {
- envTag := fieldType.Tag.Get("env")
- defaultValue := fieldType.Tag.Get("default")
-
- if fallback, fallbackErr := time.ParseDuration(defaultValue); fallbackErr == nil {
- slog.WarnContext(ctx, "Invalid duration for config field, using tagged default",
- "reason", reason,
- "field", envTag,
- "value", value,
- "default", defaultValue)
- field.SetInt(int64(fallback))
- } else {
- slog.WarnContext(ctx, "Invalid duration for config field and invalid tagged default",
- "reason", reason,
- "field", envTag,
- "value", value,
- "default", defaultValue)
- }
+ d, err := time.ParseDuration(value)
+ if err == nil && d > 0 {
+ field.SetInt(int64(d))
+ return
}
-
- if d, err := time.ParseDuration(value); err == nil {
- if d > 0 {
- field.SetInt(int64(d))
- } else {
- applyDurationDefault("Non-positive duration for config field")
- }
- } else {
- applyDurationDefault("Invalid duration for config field")
+ reason := kit.Ternary(err == nil, "Non-positive duration for config field", "Invalid duration for config field")
+ envTag, defaultValue := fieldType.Tag.Get("env"), fieldType.Tag.Get("default")
+ fallback, fallbackErr := time.ParseDuration(defaultValue)
+ if fallbackErr != nil {
+ slog.WarnContext(ctx, "Invalid duration for config field and invalid tagged default", "reason", reason, "field", envTag, "value", value, "default", defaultValue)
+ return
}
+ slog.WarnContext(ctx, "Invalid duration for config field, using tagged default", "reason", reason, "field", envTag, "value", value, "default", defaultValue)
+ field.SetInt(int64(fallback))
return
}
- // Handle custom types based on underlying kind
+ // String-based types like AppEnvironment; os.FileMode is handled as Uint32 above.
if field.Type().ConvertibleTo(reflect.TypeFor[string]()) {
- // String-based types like AppEnvironment
field.Set(reflect.ValueOf(value).Convert(field.Type()))
- } else if field.Type() == reflect.TypeFor[os.FileMode]() {
- // os.FileMode
- if i, err := strconv.ParseUint(value, 8, 32); err == nil {
- field.Set(reflect.ValueOf(os.FileMode(i)))
- }
}
}
@@ -463,10 +436,7 @@ func (c *Config) MaskSensitive() map[string]any {
envTag := cmp.Or(fieldType.Tag.Get("env"), fieldType.Name)
// Fields with "file" option are considered sensitive
- optionsTag := fieldType.Tag.Get("options")
- isSensitive := strings.Contains(optionsTag, "file")
-
- if isSensitive {
+ if hasOption(fieldType, "file") {
// Mask sensitive values
strVal := fmt.Sprintf("%v", field.Interface())
result[envTag] = kit.Ternary(strVal != "", "****", "(empty)")
diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go
index 2c200ca28..74a3146d6 100644
--- a/backend/internal/config/config_test.go
+++ b/backend/internal/config/config_test.go
@@ -557,7 +557,7 @@ func TestConfig_JWTRefreshExpiry(t *testing.T) {
})
}
-func TestSetFieldValueInternal_DurationUsesFieldTagDefault(t *testing.T) {
+func TestSetFieldValue_DurationUsesFieldTagDefault(t *testing.T) {
type durationConfig struct {
SessionTimeout time.Duration `env:"SESSION_TIMEOUT" default:"2h"`
}
@@ -568,7 +568,7 @@ func TestSetFieldValueInternal_DurationUsesFieldTagDefault(t *testing.T) {
fieldType, ok := v.Type().FieldByName("SessionTimeout")
require.True(t, ok)
- setFieldValueInternal(field, fieldType, "invalid-duration")
+ setFieldValue(field, fieldType, "invalid-duration")
assert.Equal(t, 2*time.Hour, cfg.SessionTimeout)
}
diff --git a/backend/internal/container/model.go b/backend/internal/container/model.go
new file mode 100644
index 000000000..a6680696f
--- /dev/null
+++ b/backend/internal/container/model.go
@@ -0,0 +1,15 @@
+package container
+
+import "time"
+
+// redeployJournalKeyPrefix keys redeploy journals in the KV store by the original container ID.
+const redeployJournalKeyPrefix = "container_redeploy_journal:"
+
+// RedeployJournal records a container Arcane renamed to a backup name, so recovery only touches its own backups.
+type RedeployJournal struct {
+ OriginalID string `json:"originalId"`
+ OriginalName string `json:"originalName"`
+ BackupName string `json:"backupName"`
+ NewName string `json:"newName"`
+ StartedAt time.Time `json:"startedAt"`
+}
diff --git a/backend/internal/container/service.go b/backend/internal/container/service.go
index 2e0093769..e2e245c89 100644
--- a/backend/internal/container/service.go
+++ b/backend/internal/container/service.go
@@ -3,6 +3,7 @@ package container
import (
"cmp"
"context"
+ "encoding/json/v2"
"errors"
"fmt"
"io"
@@ -42,12 +43,14 @@ import (
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/iconcatalog"
@@ -57,6 +60,12 @@ const (
containerGroupByProject = "project"
containerNoProjectGroup = "No Project"
containerIconMetadataTTL = 5 * time.Second
+ // leaseWaitTimeout bounds how long a container operation waits for one already running on the same container.
+ leaseWaitTimeout = 30 * time.Second
+ // rollbackTimeout bounds a redeploy rollback, which runs detached from the request so it completes after cancellation.
+ rollbackTimeout = 2 * timeouts.DefaultDockerAPI
+ // redeployBackupInfix joins a container's name and a Unix timestamp to name the original kept during a redeploy.
+ redeployBackupInfix = "-arcane-redeploy-"
)
type ContainerService struct {
@@ -65,8 +74,11 @@ type ContainerService struct {
imageService *image.ImageService
settingsService *settings.SettingsService
projectService *project.ProjectService
+ kvService *kv.KVService
iconMetaCache *hot.HotCache[string, projects.ArcaneComposeMetadata]
stats *stats.Service
+ // opLock serializes mutating operations on the same container.
+ opLock utils.KeyedMutex
}
type ContainerListResult struct {
@@ -81,7 +93,7 @@ type containerLifecycleAction struct {
eventType event.EventType
metadata database.JSON
warnOnLogError bool
- runContainerAction func(*client.Client) error
+ runContainerAction func(context.Context) error
}
type ExecSession struct {
@@ -105,6 +117,7 @@ func NewContainerService(
imageService *image.ImageService,
settingsService *settings.SettingsService,
projectService *project.ProjectService,
+ kvService *kv.KVService,
) *ContainerService {
return &ContainerService{
eventService: eventService,
@@ -112,6 +125,7 @@ func NewContainerService(
imageService: imageService,
settingsService: settingsService,
projectService: projectService,
+ kvService: kvService,
iconMetaCache: hot.NewHotCache[string, projects.ArcaneComposeMetadata](hot.LRU, 1024).
WithTTL(containerIconMetadataTTL).
WithJanitor().
@@ -162,125 +176,133 @@ func buildCleanNetworkingConfig(containerInspect container.InspectResponse, apiV
func (s *ContainerService) pullRedeployImage(
ctx context.Context,
- dockerClient *client.Client,
imageName, containerID, containerName, action string,
credentials []containerregistry.Credential,
user usertypes.Actor,
) error {
- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))
- defer pullCancel()
-
- pullOptions, authErr := s.imageService.PullOptionsWithAuth(ctx, imageName, credentials)
- if authErr != nil {
- slog.WarnContext(
- ctx, "failed to get registry authentication for container recreate pull; proceeding without auth",
- "image", imageName,
- "error", authErr.Error(),
- )
- pullOptions = client.ImagePullOptions{}
- }
-
- defer s.eventService.BeginDockerResourceSuppressionWindow("image", "", imageName)()
- reader, pullErr := dockerClient.ImagePull(pullCtx, imageName, pullOptions)
- if pullErr != nil && image.ShouldRetryAnonymousPull(pullOptions, pullErr) {
- slog.WarnContext(
- ctx, "container recreate image pull failed with registry auth; retrying anonymously",
- "image", imageName,
- "error", pullErr.Error(),
- )
- pullOptions = client.ImagePullOptions{}
- reader, pullErr = dockerClient.ImagePull(pullCtx, imageName, pullOptions)
- }
+ progressWriter, _ := ctx.Value(types.ProgressWriterKey{}).(io.Writer)
+ pullErr := s.imageService.Pull(ctx, image.PullRequest{
+ Ref: imageName,
+ Progress: progressWriter,
+ User: user,
+ Credentials: credentials,
+ })
if pullErr != nil {
- if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", pullErr, database.JSON{
- "action": action,
- "step": "pull_image_timeout",
- "image": imageName,
- })
- return fmt.Errorf("image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", imageName)
- }
-
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", pullErr, database.JSON{
"action": action,
- "step": "pull_image",
+ "step": kit.Ternary(errors.Is(pullErr, context.DeadlineExceeded), "pull_image_timeout", "pull_image"),
"image": imageName,
})
- return fmt.Errorf("failed to pull image %s: %w", imageName, pullErr)
}
- defer func() { _ = reader.Close() }()
-
- progressWriter, _ := ctx.Value(types.ProgressWriterKey{}).(io.Writer)
- logWriter := docker.NewLogLineWriter(progressWriter)
- defer func() { _ = logWriter.Close() }()
+ return pullErr
+}
- streamErr := docker.RenderJSONMessageStream(reader, logWriter)
- if streamErr != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", streamErr, database.JSON{
- "action": action,
- "step": "complete_pull",
- "image": imageName,
- })
- return fmt.Errorf("failed to complete image pull: %w", streamErr)
+// RecoverInterruptedRedeploys resolves journaled redeploy backups a restart left behind: a backup whose replacement
+// is running is removed, and a backup with no replacement gets its original name back but stays stopped.
+func (s *ContainerService) RecoverInterruptedRedeploys(ctx context.Context) error {
+ entries, err := s.kvService.ListByPrefix(ctx, redeployJournalKeyPrefix)
+ if err != nil {
+ return fmt.Errorf("list redeploy journals: %w", err)
+ }
+ if len(entries) == 0 {
+ return nil
}
- return nil
-}
+ list, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
+ if err != nil {
+ return fmt.Errorf("list containers for redeploy recovery: %w", err)
+ }
-func (s *ContainerService) prepareContainerForRedeploy(
- ctx context.Context,
- dockerClient *client.Client,
- containerID, containerName, backupName string,
- wasRunning bool,
- user usertypes.Actor,
-) error {
- s.eventService.MarkDockerExpectation("container", containerID, containerName)
- if containerName != "" {
- if _, err := dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: backupName}); err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", err, database.JSON{
- "action": "redeploy",
- "step": "rename_old",
- "backupName": backupName,
- })
- return fmt.Errorf("failed to rename existing container: %w", err)
+ byName := make(map[string]container.Summary, len(list.Items))
+ for _, c := range list.Items {
+ for _, name := range c.Names {
+ byName[strings.TrimPrefix(name, "/")] = c
}
}
- if !wasRunning {
- return nil
- }
+ var firstErr error
+ var removed, restored, skipped int
+ for _, entry := range entries {
+ var journal RedeployJournal
+ if unmarshalErr := json.Unmarshal([]byte(entry.Value), &journal); unmarshalErr != nil {
+ slog.WarnContext(ctx, "Failed to decode redeploy journal", "key", entry.Key, "error", unmarshalErr)
+ continue
+ }
- _, err := dockerClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: new(30)})
- if err == nil {
- return nil
- }
+ backup, backupExists := byName[journal.BackupName]
+ if !backupExists {
+ s.clearRedeployJournal(ctx, journal.OriginalID)
+ continue
+ }
+ if backup.ID != journal.OriginalID {
+ slog.WarnContext(ctx, "Skipping redeploy backup recovery: container is not the one Arcane renamed", "container", journal.BackupName, "id", backup.ID, "journaledId", journal.OriginalID)
+ s.clearRedeployJournal(ctx, journal.OriginalID)
+ skipped++
+ continue
+ }
- if containerName != "" {
- if _, renameErr := dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: containerName}); renameErr != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", renameErr, database.JSON{
- "action": "redeploy",
- "step": "restore_name_after_stop_failure",
- })
+ replacement, exists := byName[cmp.Or(journal.NewName, journal.OriginalName)]
+ switch {
+ case exists && replacement.State == container.StateRunning:
+ if _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, backup.ID, client.ContainerRemoveOptions{Force: true})
+ }); removeErr != nil {
+ slog.WarnContext(ctx, "Failed to remove leftover redeploy backup", "container", journal.BackupName, "error", removeErr)
+ firstErr = cmp.Or(firstErr, fmt.Errorf("remove redeploy backup %s: %w", journal.BackupName, removeErr))
+ continue
+ }
+ s.clearRedeployJournal(ctx, journal.OriginalID)
+ removed++
+ case exists:
+ slog.WarnContext(ctx, "Leaving redeploy backup in place: replacement exists but is not running",
+ "container", journal.BackupName,
+ "replacement", cmp.Or(journal.NewName, journal.OriginalName),
+ "state", replacement.State)
+ skipped++
+ default:
+ if _, renameErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRenameResult, error) {
+ return dockerClient.ContainerRename(ctx, backup.ID, client.ContainerRenameOptions{NewName: journal.OriginalName})
+ }); renameErr != nil {
+ slog.WarnContext(ctx, "Failed to restore leftover redeploy backup", "container", journal.BackupName, "error", renameErr)
+ firstErr = cmp.Or(firstErr, fmt.Errorf("restore redeploy backup %s: %w", journal.BackupName, renameErr))
+ continue
+ }
+ s.clearRedeployJournal(ctx, journal.OriginalID)
+ byName[journal.OriginalName] = backup
+ restored++
+ slog.WarnContext(ctx, "Restored original container from interrupted redeploy; start it manually if it should be running", "container", journal.OriginalName, "state", backup.State)
}
}
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", err, database.JSON{
- "action": "redeploy",
- "step": "stop",
- })
- return fmt.Errorf("failed to stop container: %w", err)
+ if removed+restored+skipped > 0 {
+ slog.InfoContext(ctx, "Recovered interrupted container redeploys", "removed", removed, "restored", restored, "skipped", skipped)
+ }
+ return firstErr
+}
+
+// clearRedeployJournal drops the journal once its backup is resolved; a leftover entry is retried at startup.
+func (s *ContainerService) clearRedeployJournal(ctx context.Context, originalID string) {
+ if err := s.kvService.Delete(ctx, redeployJournalKeyPrefix+originalID); err != nil {
+ slog.WarnContext(ctx, "Failed to clear redeploy journal", "containerId", originalID, "error", err)
+ }
}
func (s *ContainerService) restoreContainerAfterRedeployFailure(
ctx context.Context,
- dockerClient *client.Client,
containerID, containerName, backupName, failedStep string,
wasRunning bool,
user usertypes.Actor,
) {
+ ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rollbackTimeout)
+ defer cancel()
+
s.eventService.MarkDockerExpectation("container", containerID, containerName)
if wasRunning {
- if _, startErr := dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{}); startErr != nil {
+ if _, startErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{})
+ }); startErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", startErr, database.JSON{
"action": "redeploy",
"step": "restore_start_original",
@@ -293,21 +315,77 @@ func (s *ContainerService) restoreContainerAfterRedeployFailure(
return
}
- if _, renameErr := dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: containerName}); renameErr != nil {
+ if _, renameErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRenameResult, error) {
+ return dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: containerName})
+ }); renameErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, backupName, user.ID, user.Username, "0", renameErr, database.JSON{
"action": "redeploy",
"step": "restore_name",
"failedStep": failedStep,
})
+ return
}
+ s.clearRedeployJournal(ctx, containerID)
}
-func (s *ContainerService) runContainerLifecycleAction(ctx context.Context, containerID string, user usertypes.Actor, cfg containerLifecycleAction) error {
+// renameToRedeployBackup journals the original before renaming it to its backup name, so startup
+// recovery only acts on backups Arcane created.
+func (s *ContainerService) renameToRedeployBackup(ctx context.Context, containerID, containerName, backupName, newName string, user usertypes.Actor) error {
+ journal, err := json.Marshal(RedeployJournal{
+ OriginalID: containerID,
+ OriginalName: containerName,
+ BackupName: backupName,
+ NewName: newName,
+ StartedAt: time.Now().UTC(),
+ })
+ if err != nil {
+ return fmt.Errorf("marshal redeploy journal: %w", err)
+ }
+ if setErr := s.kvService.Set(ctx, redeployJournalKeyPrefix+containerID, string(journal)); setErr != nil {
+ return fmt.Errorf("write redeploy journal: %w", setErr)
+ }
+
+ if _, renameErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRenameResult, error) {
+ return dockerClient.ContainerRename(ctx, containerID, client.ContainerRenameOptions{NewName: backupName})
+ }); renameErr != nil {
+ s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, containerName, user.ID, user.Username, "0", renameErr, database.JSON{
+ "action": "redeploy",
+ "step": "rename_old",
+ "backupName": backupName,
+ })
+ s.clearRedeployJournal(context.WithoutCancel(ctx), containerID)
+ return fmt.Errorf("failed to rename existing container: %w", renameErr)
+ }
+ return nil
+}
+
+// stopContainer waits out a 30s grace period, so it runs without the Docker API deadline.
+func (s *ContainerService) stopContainer(ctx context.Context, containerID string) error {
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{"action": cfg.action})
return fmt.Errorf("failed to connect to Docker: %w", err)
}
+ _, err = dockerClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: new(30)})
+ return err
+}
+
+// lockContainer takes the container's operation lease, waiting up to leaseWaitTimeout for an operation already running on it.
+func (s *ContainerService) lockContainer(ctx context.Context, op, containerID string) (func(), error) {
+ waitCtx, cancel := context.WithTimeout(ctx, leaseWaitTimeout)
+ defer cancel()
+ unlock, err := s.opLock.LockContext(waitCtx, "0:container:"+containerID)
+ if err != nil {
+ return nil, fmt.Errorf("%s %s: waiting for in-progress operation: %w", op, containerID, err)
+ }
+ return unlock, nil
+}
+
+func (s *ContainerService) runContainerLifecycleAction(ctx context.Context, containerID string, user usertypes.Actor, cfg containerLifecycleAction) error {
+ unlock, err := s.lockContainer(ctx, cfg.action+" container", containerID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
metadata := database.JSON{
"action": cfg.action,
@@ -324,7 +402,7 @@ func (s *ContainerService) runContainerLifecycleAction(ctx context.Context, cont
}
defer s.eventService.BeginDockerResourceSuppressionWindow("container", containerID, "")()
- err = cfg.runContainerAction(dockerClient)
+ err = cfg.runContainerAction(ctx)
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{"action": cfg.action})
}
@@ -336,8 +414,10 @@ func (s *ContainerService) StartContainer(ctx context.Context, containerID strin
action: "start",
eventType: event.EventTypeContainerStart,
warnOnLogError: true,
- runContainerAction: func(dockerClient *client.Client) error {
- _, err := dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{})
+ runContainerAction: func(ctx context.Context) error {
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, containerID, client.ContainerStartOptions{})
+ })
return err
},
})
@@ -347,9 +427,8 @@ func (s *ContainerService) StopContainer(ctx context.Context, containerID string
return s.runContainerLifecycleAction(ctx, containerID, user, containerLifecycleAction{
action: "stop",
eventType: event.EventTypeContainerStop,
- runContainerAction: func(dockerClient *client.Client) error {
- _, err := dockerClient.ContainerStop(ctx, containerID, client.ContainerStopOptions{Timeout: new(30)})
- return err
+ runContainerAction: func(ctx context.Context) error {
+ return s.stopContainer(ctx, containerID)
},
})
}
@@ -358,8 +437,12 @@ func (s *ContainerService) RestartContainer(ctx context.Context, containerID str
return s.runContainerLifecycleAction(ctx, containerID, user, containerLifecycleAction{
action: "restart",
eventType: event.EventTypeContainerRestart,
- runContainerAction: func(dockerClient *client.Client) error {
- _, err := dockerClient.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})
+ runContainerAction: func(ctx context.Context) error {
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+ _, err = dockerClient.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})
return err
},
})
@@ -373,8 +456,10 @@ func (s *ContainerService) KillContainer(ctx context.Context, containerID, signa
eventType: event.EventTypeContainerKill,
metadata: database.JSON{"signal": signal},
warnOnLogError: true,
- runContainerAction: func(dockerClient *client.Client) error {
- _, err := dockerClient.ContainerKill(ctx, containerID, client.ContainerKillOptions{Signal: signal})
+ runContainerAction: func(ctx context.Context) error {
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerKillResult, error) {
+ return dockerClient.ContainerKill(ctx, containerID, client.ContainerKillOptions{Signal: signal})
+ })
return err
},
})
@@ -386,8 +471,10 @@ func (s *ContainerService) PauseContainer(ctx context.Context, containerID strin
action: "pause",
eventType: event.EventTypeContainerPause,
warnOnLogError: true,
- runContainerAction: func(dockerClient *client.Client) error {
- _, err := dockerClient.ContainerPause(ctx, containerID, client.ContainerPauseOptions{})
+ runContainerAction: func(ctx context.Context) error {
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerPauseResult, error) {
+ return dockerClient.ContainerPause(ctx, containerID, client.ContainerPauseOptions{})
+ })
return err
},
})
@@ -399,8 +486,10 @@ func (s *ContainerService) UnpauseContainer(ctx context.Context, containerID str
action: "unpause",
eventType: event.EventTypeContainerUnpause,
warnOnLogError: true,
- runContainerAction: func(dockerClient *client.Client) error {
- _, err := dockerClient.ContainerUnpause(ctx, containerID, client.ContainerUnpauseOptions{})
+ runContainerAction: func(ctx context.Context) error {
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerUnpauseResult, error) {
+ return dockerClient.ContainerUnpause(ctx, containerID, client.ContainerUnpauseOptions{})
+ })
return err
},
})
@@ -451,16 +540,15 @@ func (s *ContainerService) CommitContainer(ctx context.Context, containerID stri
}
func (s *ContainerService) RedeployContainer(ctx context.Context, containerID string, user usertypes.Actor) (string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ unlock, err := s.lockContainer(ctx, "redeploy container", containerID)
if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{
- "action": "redeploy",
- "step": "get_client",
- })
- return "", fmt.Errorf("failed to connect to Docker: %w", err)
+ return "", err
}
+ defer unlock()
- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ })
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{
"action": "redeploy",
@@ -480,7 +568,9 @@ func (s *ContainerService) RedeployContainer(ctx context.Context, containerID st
}
containerName := strings.TrimPrefix(containerInfo.Name, "/")
- apiVersion := compat.DetectDockerAPIVersion(ctx, dockerClient)
+ apiVersion, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
+ return compat.DetectDockerAPIVersion(ctx, dockerClient), nil
+ })
currentContainerID, currentContainerErr := cgroup.CurrentContainerID()
if labels.ShouldDisableArcaneServerRedeploy(containerInfo.Config.Labels, containerInfo.ID, currentContainerID, currentContainerErr) {
@@ -550,14 +640,13 @@ func (s *ContainerService) RedeployContainer(ctx context.Context, containerID st
}
if containerInfo.Config.Image != "" {
- if pullRedeployImageErr := s.pullRedeployImage(ctx, dockerClient, containerInfo.Config.Image, containerID, containerName, "redeploy", nil, user); pullRedeployImageErr != nil {
+ if pullRedeployImageErr := s.pullRedeployImage(ctx, containerInfo.Config.Image, containerID, containerName, "redeploy", nil, user); pullRedeployImageErr != nil {
return "", pullRedeployImageErr
}
}
return s.recreateContainer(
ctx,
- dockerClient,
containerInfo,
"redeploy",
containerName,
@@ -574,7 +663,6 @@ func (s *ContainerService) RedeployContainer(ctx context.Context, containerID st
// rename-as-backup so the original can be restored when create or start fails. Shared by redeploy and edit.
func (s *ContainerService) recreateContainer(
ctx context.Context,
- dockerClient *client.Client,
containerInfo container.InspectResponse,
action, newName string,
newConfig *container.Config,
@@ -583,10 +671,7 @@ func (s *ContainerService) recreateContainer(
apiVersion string,
eventType event.EventType,
user usertypes.Actor,
-) (
- string,
- error,
-) {
+) (string, error) {
containerName := strings.TrimPrefix(containerInfo.Name, "/")
wasRunning := kit.FromPtr(containerInfo.State).Running
@@ -595,28 +680,25 @@ func (s *ContainerService) recreateContainer(
stopAfterCreate := wasRunning && kit.FromPtr(containerInfo.HostConfig).AutoRemove
defer s.eventService.BeginDockerResourceSuppressionWindow("container", containerInfo.ID, containerName)()
- backupName := fmt.Sprintf("%s-arcane-redeploy-%d", cmp.Or(containerName, func() string {
- if len(containerInfo.ID) < 12 {
- return "arcane-redeploy"
- }
- return "arcane-redeploy-" + containerInfo.ID[:12]
- }()), time.Now().Unix())
+ backupName := fmt.Sprintf("%s%s%d", cmp.Or(containerName, "arcane-redeploy-"+containerInfo.ID[:min(12, len(containerInfo.ID))]), redeployBackupInfix, time.Now().Unix())
// Stopping the original made the daemon delete it, so rebuild it from the inspected config.
// A returned error means the workload is down and is joined into the start error.
- restoreAutoRemoveOriginal := func() error {
+ restoreAutoRemoveOriginal := func(ctx context.Context) error {
originalConfig := *containerInfo.Config
if len(containerInfo.ID) >= 12 && originalConfig.Hostname == containerInfo.ID[:12] {
originalConfig.Hostname = "" // keep: copying Docker's 24-field container.Config to clear one field is unreadable
}
defer s.eventService.BeginDockerResourceSuppressionWindow("container", "", containerName)()
- restoreResp, restoreCreateErr := compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{
- Config: &originalConfig,
- HostConfig: containerInfo.HostConfig,
- NetworkingConfig: buildCleanNetworkingConfig(containerInfo, apiVersion),
- Name: containerName,
- }, apiVersion)
+ restoreResp, restoreCreateErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{
+ Config: &originalConfig,
+ HostConfig: containerInfo.HostConfig,
+ NetworkingConfig: buildCleanNetworkingConfig(containerInfo, apiVersion),
+ Name: containerName,
+ }, apiVersion)
+ })
if restoreCreateErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", restoreCreateErr, database.JSON{
"action": "redeploy",
@@ -626,7 +708,9 @@ func (s *ContainerService) recreateContainer(
}
defer s.eventService.BeginDockerResourceSuppressionWindow("container", restoreResp.ID, containerName)()
- if _, restoreStartErr := dockerClient.ContainerStart(ctx, restoreResp.ID, client.ContainerStartOptions{}); restoreStartErr != nil {
+ if _, restoreStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, restoreResp.ID, client.ContainerStartOptions{})
+ }); restoreStartErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", restoreResp.ID, containerName, user.ID, user.Username, "0", restoreStartErr, database.JSON{
"action": "redeploy",
"step": "restore_start_original",
@@ -643,8 +727,21 @@ func (s *ContainerService) recreateContainer(
return nil
}
- if err := s.prepareContainerForRedeploy(ctx, dockerClient, containerInfo.ID, containerName, backupName, wasRunning && !stopAfterCreate, user); err != nil {
- return "", err
+ s.eventService.MarkDockerExpectation("container", containerInfo.ID, containerName)
+ if containerName != "" {
+ if err := s.renameToRedeployBackup(ctx, containerInfo.ID, containerName, backupName, newName, user); err != nil {
+ return "", err
+ }
+ }
+ if wasRunning && !stopAfterCreate {
+ if err := s.stopContainer(ctx, containerInfo.ID); err != nil {
+ s.restoreContainerAfterRedeployFailure(ctx, containerInfo.ID, containerName, backupName, "stop", false, user)
+ s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", err, database.JSON{
+ "action": "redeploy",
+ "step": "stop",
+ })
+ return "", fmt.Errorf("failed to stop container: %w", err)
+ }
}
if len(containerInfo.ID) >= 12 && newConfig.Hostname == containerInfo.ID[:12] {
@@ -652,14 +749,16 @@ func (s *ContainerService) recreateContainer(
}
defer s.eventService.BeginDockerResourceSuppressionWindow("container", "", newName)()
- createResp, err := compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{
- Config: newConfig,
- HostConfig: newHostConfig,
- NetworkingConfig: networkingConfig,
- Name: newName,
- }, apiVersion)
+ createResp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return compat.ContainerCreateWithCompatibilityForAPIVersion(ctx, dockerClient, client.ContainerCreateOptions{
+ Config: newConfig,
+ HostConfig: newHostConfig,
+ NetworkingConfig: networkingConfig,
+ Name: newName,
+ }, apiVersion)
+ })
if err != nil {
- s.restoreContainerAfterRedeployFailure(ctx, dockerClient, containerInfo.ID, containerName, backupName, "create", wasRunning && !stopAfterCreate, user)
+ s.restoreContainerAfterRedeployFailure(ctx, containerInfo.ID, containerName, backupName, "create", wasRunning && !stopAfterCreate, user)
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", err, database.JSON{
"action": action,
"step": "create",
@@ -671,14 +770,16 @@ func (s *ContainerService) recreateContainer(
defer s.eventService.BeginDockerResourceSuppressionWindow("container", createResp.ID, newName)()
if stopAfterCreate {
- if _, containerStopErr := dockerClient.ContainerStop(ctx, containerInfo.ID, client.ContainerStopOptions{Timeout: new(30)}); containerStopErr != nil {
- if _, removeErr := dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true}); removeErr != nil {
+ if containerStopErr := s.stopContainer(ctx, containerInfo.ID); containerStopErr != nil {
+ if _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true})
+ }); removeErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", createResp.ID, containerName, user.ID, user.Username, "0", removeErr, database.JSON{
"action": action,
"step": "cleanup_failed_stop",
})
}
- s.restoreContainerAfterRedeployFailure(ctx, dockerClient, containerInfo.ID, containerName, backupName, "stop", false, user)
+ s.restoreContainerAfterRedeployFailure(ctx, containerInfo.ID, containerName, backupName, "stop", false, user)
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerInfo.ID, containerName, user.ID, user.Username, "0", containerStopErr, database.JSON{
"action": action,
"step": "stop",
@@ -689,11 +790,17 @@ func (s *ContainerService) recreateContainer(
restartPolicies := []container.RestartPolicyMode{container.RestartPolicyAlways, container.RestartPolicyUnlessStopped, container.RestartPolicyOnFailure}
if wasRunning || slices.Contains(restartPolicies, kit.FromPtr(containerInfo.HostConfig).RestartPolicy.Name) {
- if _, err = dockerClient.ContainerStart(ctx, createResp.ID, client.ContainerStartOptions{}); err != nil {
+ if _, err = s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, createResp.ID, client.ContainerStartOptions{})
+ }); err != nil {
// Remove the replacement and roll back to the original. A deferred-stop auto-remove original
// is rebuilt from its inspected config; a failed restore joins the start error.
+ rollbackCtx, cancelRollback := context.WithTimeout(context.WithoutCancel(ctx), rollbackTimeout)
+ defer cancelRollback()
s.eventService.MarkDockerExpectation("container", createResp.ID, containerName)
- _, removeErr := dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true})
+ _, removeErr := s.dockerService.Call(rollbackCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, createResp.ID, client.ContainerRemoveOptions{Force: true})
+ })
if removeErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", createResp.ID, containerName, user.ID, user.Username, "0", removeErr, database.JSON{
"action": action,
@@ -701,11 +808,12 @@ func (s *ContainerService) recreateContainer(
})
}
if stopAfterCreate {
- if restoreErr := restoreAutoRemoveOriginal(); restoreErr != nil {
+ if restoreErr := restoreAutoRemoveOriginal(rollbackCtx); restoreErr != nil {
err = errors.Join(err, kit.Ternary(removeErr != nil, fmt.Errorf("failed to remove replacement container: %w", removeErr), nil), restoreErr)
}
+ s.clearRedeployJournal(rollbackCtx, containerInfo.ID)
} else {
- s.restoreContainerAfterRedeployFailure(ctx, dockerClient, containerInfo.ID, containerName, backupName, "start", wasRunning, user)
+ s.restoreContainerAfterRedeployFailure(rollbackCtx, containerInfo.ID, containerName, backupName, "start", wasRunning, user)
}
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", createResp.ID, containerName, user.ID, user.Username, "0", err, database.JSON{
"action": action,
@@ -725,20 +833,25 @@ func (s *ContainerService) recreateContainer(
"image", newConfig.Image,
)
- // After a deferred stop the daemon has already auto-removed the original.
- if !stopAfterCreate {
- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerInfo.ID, client.ContainerRemoveOptions{
+ // After a deferred stop the daemon has already auto-removed the original. A failed removal
+ // keeps the journal so startup recovery removes the backup.
+ if stopAfterCreate {
+ s.clearRedeployJournal(ctx, containerInfo.ID)
+ } else if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, containerInfo.ID, client.ContainerRemoveOptions{
Force: true,
RemoveVolumes: false,
RemoveLinks: false,
- }); containerRemoveErr != nil {
- slog.WarnContext(
- ctx, "failed to remove old container after successful recreate",
- "containerId", containerInfo.ID,
- "backupName", backupName,
- "error", containerRemoveErr,
- )
- }
+ })
+ }); containerRemoveErr != nil {
+ slog.WarnContext(
+ ctx, "failed to remove old container after successful recreate",
+ "containerId", containerInfo.ID,
+ "backupName", backupName,
+ "error", containerRemoveErr,
+ )
+ } else {
+ s.clearRedeployJournal(ctx, containerInfo.ID)
}
if logErr := s.eventService.LogContainerEvent(ctx, eventType, createResp.ID, newName, user.ID, user.Username, "0", database.JSON{
@@ -960,12 +1073,9 @@ func buildEditNetworkingConfig(containerInspect container.InspectResponse, req *
// GetContainerEditConfig returns the purpose-built DTO backing the container
// edit form.
func (s *ContainerService) GetContainerEditConfig(ctx context.Context, containerID string) (containertypes.EditConfig, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return containertypes.EditConfig{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ })
if err != nil {
return containertypes.EditConfig{}, fmt.Errorf("failed to inspect container: %w", err)
}
@@ -982,16 +1092,15 @@ func (s *ContainerService) GetContainerEditConfig(ctx context.Context, container
// inspected config and recreates it. Sections not owned by the edit form are
// preserved from the existing container.
func (s *ContainerService) EditContainer(ctx context.Context, containerID string, req containertypes.Edit, user usertypes.Actor) (string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ unlock, err := s.lockContainer(ctx, "edit container", containerID)
if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{
- "action": "edit",
- "step": "get_client",
- })
- return "", fmt.Errorf("failed to connect to Docker: %w", err)
+ return "", err
}
+ defer unlock()
- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ })
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", containerID, "", user.ID, user.Username, "0", err, database.JSON{
"action": "edit",
@@ -1023,12 +1132,16 @@ func (s *ContainerService) EditContainer(ctx context.Context, containerID string
newName := cmp.Or(strings.TrimPrefix(strings.TrimSpace(kit.FromPtr(req.Name)), "/"), containerName)
if newName != containerName {
- if _, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, newName, client.ContainerInspectOptions{}); inspectErr == nil {
+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, newName, client.ContainerInspectOptions{})
+ }); inspectErr == nil {
return "", fmt.Errorf("container name %s: %w", newName, common.ErrContainerNameTaken)
}
}
- apiVersion := compat.DetectDockerAPIVersion(ctx, dockerClient)
+ apiVersion, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
+ return compat.DetectDockerAPIVersion(ctx, dockerClient), nil
+ })
newHostConfig := kit.FromPtr(containerInfo.HostConfig)
newConfig, err := applyEditToContainerConfig(*containerInfo.Config, newHostConfig.PortBindings, req)
@@ -1046,23 +1159,22 @@ func (s *ContainerService) EditContainer(ctx context.Context, containerID string
// Unlike redeploy, edit only pulls when the effective image is missing
// locally so an unrelated config change never surprise-upgrades the image.
if newConfig.Image != "" {
- if _, inspectErr := dockerClient.ImageInspect(ctx, newConfig.Image); inspectErr != nil {
- if pullRedeployImageErr := s.pullRedeployImage(ctx, dockerClient, newConfig.Image, containerID, containerName, "edit", req.Credentials, user); pullRedeployImageErr != nil {
+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, newConfig.Image)
+ }); inspectErr != nil {
+ if pullRedeployImageErr := s.pullRedeployImage(ctx, newConfig.Image, containerID, containerName, "edit", req.Credentials, user); pullRedeployImageErr != nil {
return "", pullRedeployImageErr
}
}
}
- return s.recreateContainer(ctx, dockerClient, containerInfo, "edit", newName, &newConfig, &newHostConfig, networkingConfig, apiVersion, event.EventTypeContainerUpdate, user)
+ return s.recreateContainer(ctx, containerInfo, "edit", newName, &newConfig, &newHostConfig, networkingConfig, apiVersion, event.EventTypeContainerUpdate, user)
}
func (s *ContainerService) GetContainerByReference(ctx context.Context, ref string) (*container.InspectResponse, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- containerInspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, ref, client.ContainerInspectOptions{})
+ containerInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, ref, client.ContainerInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("container not found: %w", err)
}
@@ -1126,21 +1238,17 @@ var containerProcessesPsArgs = [][]string{
// GetContainerProcesses returns Docker's `top` snapshot for a container.
func (s *ContainerService) GetContainerProcesses(ctx context.Context, containerID string) (containertypes.Processes, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return containertypes.Processes{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- topCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
- defer cancel()
-
- var result client.ContainerTopResult
- for _, args := range containerProcessesPsArgs {
- result, err = dockerClient.ContainerTop(topCtx, containerID, client.ContainerTopOptions{Arguments: args})
- if err == nil || errdefs.IsNotFound(err) || errdefs.IsConflict(err) || topCtx.Err() != nil {
- break
+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerTopResult, error) {
+ var result client.ContainerTopResult
+ var err error
+ for _, args := range containerProcessesPsArgs {
+ result, err = dockerClient.ContainerTop(ctx, containerID, client.ContainerTopOptions{Arguments: args})
+ if err == nil || errdefs.IsNotFound(err) || errdefs.IsConflict(err) || ctx.Err() != nil {
+ break
+ }
}
- }
+ return result, err
+ })
if err != nil {
return containertypes.Processes{}, common.Classify(func() error {
switch {
@@ -1176,31 +1284,18 @@ func (s *ContainerService) GetContainerNameByID(ctx context.Context, id string)
}
func (s *ContainerService) DeleteContainer(ctx context.Context, containerID string, force, removeVolumes bool, user usertypes.Actor) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ unlock, err := s.lockContainer(ctx, "delete container", containerID)
if err != nil {
- s.eventService.LogErrorEvent(
- ctx,
- event.EventTypeContainerError,
- "container",
- containerID,
- "",
- user.ID,
- user.Username,
- "0",
- err,
- database.JSON{
- "action": "delete",
- "force": force,
- "removeVolumes": removeVolumes,
- },
- )
- return fmt.Errorf("failed to connect to Docker: %w", err)
+ return err
}
+ defer unlock()
// Get container mounts before deletion if we need to remove volumes
var volumesToRemove []string
if removeVolumes {
- containerJSON, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ containerJSON, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ })
if inspectErr == nil {
for _, mount := range containerJSON.Container.Mounts {
// Only collect named volumes (not bind mounts or tmpfs)
@@ -1214,17 +1309,19 @@ func (s *ContainerService) DeleteContainer(ctx context.Context, containerID stri
defer s.eventService.BeginDockerResourceSuppressionWindow("container", containerID, "")()
volumeReleases := make([]func(), 0, len(volumesToRemove))
defer func() {
- for i := len(volumeReleases) - 1; i >= 0; i-- {
- volumeReleases[i]()
+ for _, release := range slices.Backward(volumeReleases) {
+ release()
}
}()
for _, volumeName := range volumesToRemove {
volumeReleases = append(volumeReleases, s.eventService.BeginDockerResourceSuppressionWindow("volume", volumeName, volumeName))
}
- _, err = dockerClient.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{
- Force: force,
- RemoveVolumes: removeVolumes,
- RemoveLinks: false,
+ _, err = s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, containerID, client.ContainerRemoveOptions{
+ Force: force,
+ RemoveVolumes: removeVolumes,
+ RemoveLinks: false,
+ })
})
if err != nil {
s.eventService.LogErrorEvent(
@@ -1248,7 +1345,9 @@ func (s *ContainerService) DeleteContainer(ctx context.Context, containerID stri
// Remove named volumes if requested
for _, volumeName := range volumesToRemove {
- if _, removeErr := dockerClient.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: false}); removeErr != nil {
+ if _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeRemoveResult, error) {
+ return dockerClient.VolumeRemove(ctx, volumeName, client.VolumeRemoveOptions{Force: false})
+ }); removeErr != nil {
// Log but don't fail if volume removal fails (might be in use by another container)
s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", volumeName, "", user.ID, user.Username, "0", removeErr, database.JSON{"action": "delete", "container": containerID})
}
@@ -1277,25 +1376,23 @@ func (s *ContainerService) CreateContainer(
*container.InspectResponse,
error,
) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeContainerError, "container", "", containerName, user.ID, user.Username, "0", err, database.JSON{"action": "create", "image": config.Image})
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, inspectErr := dockerClient.ImageInspect(ctx, config.Image); inspectErr != nil {
+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, config.Image)
+ }); inspectErr != nil {
// Image not found locally, need to pull it
- if pullErr := s.pullRedeployImage(ctx, dockerClient, config.Image, "", containerName, "create", credentials, user); pullErr != nil {
+ if pullErr := s.pullRedeployImage(ctx, config.Image, "", containerName, "create", credentials, user); pullErr != nil {
return nil, pullErr
}
}
defer s.eventService.BeginDockerResourceSuppressionWindow("container", "", containerName)()
- resp, err := compat.ContainerCreateWithCompatibility(ctx, dockerClient, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
- NetworkingConfig: networkingConfig,
- Name: containerName,
+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return compat.ContainerCreateWithCompatibility(ctx, dockerClient, client.ContainerCreateOptions{
+ Config: config,
+ HostConfig: hostConfig,
+ NetworkingConfig: networkingConfig,
+ Name: containerName,
+ })
})
if err != nil {
s.eventService.LogErrorEvent(
@@ -1325,8 +1422,12 @@ func (s *ContainerService) CreateContainer(
slog.WarnContext(ctx, "could not log container stop action", "error", logErr)
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ })
s.eventService.LogErrorEvent(
ctx,
event.EventTypeContainerError,
@@ -1346,7 +1447,9 @@ func (s *ContainerService) CreateContainer(
return nil, fmt.Errorf("failed to start container: %w", containerStartErr)
}
- containerJSON, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, resp.ID, client.ContainerInspectOptions{})
+ containerJSON, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, resp.ID, client.ContainerInspectOptions{})
+ })
if err != nil {
s.eventService.LogErrorEvent(
ctx,
@@ -1985,14 +2088,16 @@ func (d *demuxedLogs) Close() error {
}
func (s *ContainerService) openLogs(ctx context.Context, containerID string, options client.ContainerLogsOptions) (io.ReadCloser, client.ContainerInspectResult, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
+ containerInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ })
if err != nil {
- return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
+ return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to inspect container for logs: %w", err)
}
- containerInspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to inspect container for logs: %w", err)
+ return nil, client.ContainerInspectResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
}
logs, err := dockerClient.ContainerLogs(ctx, containerID, options)
diff --git a/backend/internal/container/service_test.go b/backend/internal/container/service_test.go
index 5a85b2d31..e1bd2367e 100644
--- a/backend/internal/container/service_test.go
+++ b/backend/internal/container/service_test.go
@@ -41,6 +41,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
imagedomain "github.com/getarcaneapp/arcane/backend/v2/internal/image"
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
@@ -144,7 +145,7 @@ func TestContainerSummaryIconsAppliedAfterGrouping(t *testing.T) {
db := setupProjectTestDB(t)
settingsService, err := settings.NewSettingsService(t.Context(), db)
require.NoError(t, err)
- service := NewContainerService(nil, nil, nil, settingsService, project.NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil))
+ service := NewContainerService(nil, nil, nil, settingsService, project.NewProjectService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil), nil)
dockerContainers := []container.Summary{
{ID: "app", Names: []string{"/app"}, Labels: map[string]string{
"com.docker.compose.project": "media",
@@ -323,6 +324,7 @@ func TestContainerServiceCommitContainerCallsDockerAPI(t *testing.T) {
nil,
nil,
nil,
+ nil,
)
out, err := svc.CommitContainer(t.Context(), "container-1", containertypes.CommitRequest{
@@ -370,6 +372,7 @@ func TestContainerServiceCommitContainerOmitsReferenceWhenRepositoryEmpty(t *tes
nil,
nil,
nil,
+ nil,
)
out, err := svc.CommitContainer(t.Context(), "container-1", containertypes.CommitRequest{
@@ -427,7 +430,7 @@ func setupProjectTestDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&project.Project{}, &settings.SettingVariable{}, &imageupdate.ImageUpdateRecord{}, &event.Event{}))
+ require.NoError(t, db.AutoMigrate(&project.Project{}, &settings.SettingVariable{}, &imageupdate.ImageUpdateRecord{}, &event.Event{}, &kv.KVEntry{}))
return &database.DB{DB: db}
}
@@ -629,8 +632,8 @@ func TestRecreateContainerDaemonCorrelation(t *testing.T) {
w.WriteHeader(http.StatusNoContent)
}))
t.Cleanup(server.Close)
- dockerClient := newTestDockerClient(t, server)
- svc := NewContainerService(events, nil, nil, nil, nil)
+ kvService := kv.NewKVService(db)
+ svc := NewContainerService(events, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, nil, nil, kvService)
info := container.InspectResponse{
ID: "old-id",
Name: "/web",
@@ -640,7 +643,6 @@ func TestRecreateContainerDaemonCorrelation(t *testing.T) {
}
id, err := svc.recreateContainer(
t.Context(),
- dockerClient,
info,
"redeploy",
"web",
@@ -673,12 +675,171 @@ func TestRecreateContainerDaemonCorrelation(t *testing.T) {
)
require.True(t, events.ShouldSuppressDaemonEvent("container", "old-id", "", ""))
require.True(t, events.ShouldSuppressDaemonEvent("container", "new-id", "", ""))
+ journals, err := kvService.ListByPrefix(t.Context(), redeployJournalKeyPrefix)
+ require.NoError(t, err)
+ require.Empty(t, journals)
+}
+
+func TestRecreateContainerRollsBackAfterRequestCancellation(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ var mu sync.Mutex
+ var observed []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ path := dockerTestPath(r.URL.Path)
+ mu.Lock()
+ observed = append(observed, r.Method+" "+path)
+ mu.Unlock()
+ switch path {
+ case "/containers/create":
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusCreated)
+ if err := json.NewEncoder(w).Encode(map[string]any{"Id": "new-id", "Warnings": []string{}}); err != nil {
+ t.Errorf("encode container creation response: %v", err)
+ }
+ case "/containers/new-id/start":
+ cancel()
+ w.WriteHeader(http.StatusInternalServerError)
+ default:
+ w.WriteHeader(http.StatusNoContent)
+ }
+ }))
+ t.Cleanup(server.Close)
+ kvService := kv.NewKVService(db)
+ svc := NewContainerService(event.NewEventService(db, nil, nil), docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, nil, nil, kvService)
+ info := container.InspectResponse{
+ ID: "old-id",
+ Name: "/web",
+ State: &container.State{Running: true},
+ Config: &container.Config{Image: "app:latest"},
+ HostConfig: &container.HostConfig{},
+ }
+
+ _, err := svc.recreateContainer(ctx, info, "redeploy", "web", info.Config, info.HostConfig, nil, "1.41", event.EventTypeContainerDeploy, user.SystemUser)
+ require.Error(t, err)
+ require.ErrorIs(t, ctx.Err(), context.Canceled)
+
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(
+ t,
+ []string{
+ "POST /containers/old-id/rename",
+ "POST /containers/old-id/stop",
+ "POST /containers/create",
+ "POST /containers/new-id/start",
+ "DELETE /containers/new-id",
+ "POST /containers/old-id/start",
+ "POST /containers/old-id/rename",
+ },
+ observed,
+ )
+ journals, err := kvService.ListByPrefix(t.Context(), redeployJournalKeyPrefix)
+ require.NoError(t, err)
+ require.Empty(t, journals)
+}
+
+func TestRecoverInterruptedRedeploys(t *testing.T) {
+ backupName := "web" + redeployBackupInfix + "1700000000"
+ backup := container.Summary{
+ ID: "backup-id",
+ Names: []string{"/" + backupName},
+ State: container.StateExited,
+ }
+ journal := RedeployJournal{
+ OriginalID: "backup-id",
+ OriginalName: "web",
+ BackupName: backupName,
+ NewName: "web",
+ }
+ tests := []struct {
+ name string
+ journals []RedeployJournal
+ containers []container.Summary
+ want []string
+ wantJournals int
+ }{
+ {
+ name: "replacement running removes backup",
+ journals: []RedeployJournal{journal},
+ containers: []container.Summary{
+ backup,
+ {ID: "new-id", Names: []string{"/web"}, State: container.StateRunning},
+ },
+ want: []string{"DELETE /containers/backup-id?force=1"},
+ },
+ {
+ name: "missing replacement restores original name",
+ journals: []RedeployJournal{journal},
+ containers: []container.Summary{backup},
+ want: []string{"POST /containers/backup-id/rename?name=web"},
+ },
+ {
+ name: "stopped replacement leaves both",
+ journals: []RedeployJournal{journal},
+ containers: []container.Summary{
+ backup,
+ {ID: "new-id", Names: []string{"/web"}, State: container.StateExited},
+ },
+ wantJournals: 1,
+ },
+ {
+ name: "backup with a different ID is skipped",
+ journals: []RedeployJournal{journal},
+ containers: []container.Summary{
+ {ID: "other-id", Names: []string{"/" + backupName}, State: container.StateExited},
+ },
+ },
+ {
+ name: "name-matching container without a journal is left alone",
+ containers: []container.Summary{backup},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ kvService := kv.NewKVService(setupProjectTestDB(t))
+ for _, j := range tt.journals {
+ payload, err := json.Marshal(j)
+ require.NoError(t, err)
+ require.NoError(t, kvService.Set(t.Context(), redeployJournalKeyPrefix+j.OriginalID, string(payload)))
+ }
+ var mu sync.Mutex
+ var mutations []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ path := dockerTestPath(r.URL.Path)
+ if path == "/containers/json" {
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(tt.containers); err != nil {
+ t.Errorf("encode container list: %v", err)
+ }
+ return
+ }
+ mu.Lock()
+ mutations = append(mutations, r.Method+" "+path+kit.Ternary(r.URL.RawQuery != "", "?"+r.URL.RawQuery, ""))
+ mu.Unlock()
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ t.Cleanup(server.Close)
+ svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, nil, nil, kvService)
+
+ require.NoError(t, svc.RecoverInterruptedRedeploys(t.Context()))
+
+ mu.Lock()
+ defer mu.Unlock()
+ require.Equal(t, tt.want, mutations)
+ journals, err := kvService.ListByPrefix(t.Context(), redeployJournalKeyPrefix)
+ require.NoError(t, err)
+ require.Len(t, journals, tt.wantJournals)
+ })
+ }
}
func TestBuildSummariesUsesContainerTagPolicyUpdates(t *testing.T) {
settingsService, err := settings.NewSettingsService(t.Context(), setupProjectTestDB(t))
require.NoError(t, err)
- service := NewContainerService(nil, nil, nil, settingsService, nil)
+ service := NewContainerService(nil, nil, nil, settingsService, nil, nil)
strategyLabel := "com.getarcaneapp.arcane.updater.strategy"
containers := []container.Summary{
{ID: "first", Image: "app:3.1.0", ImageID: "shared-image"},
@@ -790,7 +951,7 @@ func TestContainerServiceGetContainerProcesses(t *testing.T) {
t.Cleanup(server.Close)
settingsService, err := settings.NewSettingsService(t.Context(), setupProjectTestDB(t))
require.NoError(t, err)
- svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil)
+ svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil, nil)
processes, err := svc.GetContainerProcesses(t.Context(), "container-1")
wantArgs := []string{strings.Join(containerProcessesPsArgs[0], " ")}
@@ -818,7 +979,7 @@ func TestContainerServiceGetContainerProcessesPropagatesContext(t *testing.T) {
t.Cleanup(server.Close)
settingsService, err := settings.NewSettingsService(t.Context(), setupProjectTestDB(t))
require.NoError(t, err)
- svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil)
+ svc := NewContainerService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, settingsService, nil, nil)
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
defer cancel()
@@ -966,6 +1127,7 @@ func newResourceSortService(t *testing.T, fixture *resourceStatsServer) *Contain
imagedomain.NewImageService(db, dockerService, nil, nil, nil, nil, settingsService, nil),
settingsService,
nil,
+ nil,
)
}
@@ -1309,6 +1471,7 @@ func newLogsTestService(t *testing.T, tty bool, logsBody []byte, gotQuery *url.V
nil,
nil,
nil,
+ nil,
)
}
diff --git a/backend/internal/dashboard/service.go b/backend/internal/dashboard/service.go
index 676c72e5a..fb454d799 100644
--- a/backend/internal/dashboard/service.go
+++ b/backend/internal/dashboard/service.go
@@ -467,16 +467,16 @@ func (s *DashboardService) ObserveInventory(meter metric.Meter) error {
// observeSwarm reports nodes and services only when the local engine is a swarm manager; the ping keeps non-swarm hosts cheap.
func (s *DashboardService) observeSwarm(ctx context.Context, o metric.Observer, g *inventoryGauges) {
- cli, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return
- }
- ping, err := cli.Ping(ctx, client.PingOptions{})
+ ping, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.PingResult, error) {
+ return cli.Ping(ctx, client.PingOptions{})
+ })
if err != nil || ping.SwarmStatus == nil || !ping.SwarmStatus.ControlAvailable {
return
}
- if nodes, nodeErr := cli.NodeList(ctx, client.NodeListOptions{}); nodeErr != nil {
+ if nodes, nodeErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NodeListResult, error) {
+ return cli.NodeList(ctx, client.NodeListOptions{})
+ }); nodeErr != nil {
slog.DebugContext(ctx, "Skipping swarm node metrics", "error", nodeErr)
} else {
counts := map[[2]string]int64{}
@@ -488,7 +488,9 @@ func (s *DashboardService) observeSwarm(ctx context.Context, o metric.Observer,
}
}
- services, err := cli.ServiceList(ctx, client.ServiceListOptions{Status: true})
+ services, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ServiceListResult, error) {
+ return cli.ServiceList(ctx, client.ServiceListOptions{Status: true})
+ })
if err != nil {
slog.DebugContext(ctx, "Skipping swarm service metrics", "error", err)
return
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index b0489104d..03edded9c 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -394,7 +394,7 @@ func TestDashboardService_GetSnapshot_CachesFullSnapshotsPerIconCatalog(t *testi
dockerSvc := newDashboardTestDockerService(t, settingsSvc, containers, images, nil)
require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "autoUpdateExcludedContainers", "running-app"))
- containerSvc := container.NewContainerService(nil, dockerSvc, nil, settingsSvc, nil)
+ containerSvc := container.NewContainerService(nil, dockerSvc, nil, settingsSvc, nil, nil)
svc := NewDashboardService(db, dockerSvc, containerSvc, nil, nil, settingsSvc, nil, nil, nil, nil)
defaultSnapshot, err := svc.GetSnapshot(t.Context(), DashboardActionItemsOptions{}, true)
diff --git a/backend/internal/database/database.go b/backend/internal/database/database.go
index 44241b6dd..3564192b8 100644
--- a/backend/internal/database/database.go
+++ b/backend/internal/database/database.go
@@ -96,7 +96,8 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOption
}
var gormDB *gorm.DB
- for attempt := 1; attempt <= 3; attempt++ {
+ connectStart := time.Now()
+ for attempt := 1; ; attempt++ {
if err = ctx.Err(); err != nil {
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
@@ -113,13 +114,16 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOption
if err == nil {
break
}
- slog.InfoContext(ctx, "Failed to initialize database", "attempt", attempt)
- if attempt < 3 {
- select {
- case <-time.After(3 * time.Second):
- case <-ctx.Done():
- return nil, fmt.Errorf("failed to connect to database: %w", ctx.Err())
- }
+ wait := min(time.Second<<min(attempt-1, 4), 15*time.Second)
+ if time.Since(connectStart)+wait > time.Minute {
+ err = fmt.Errorf("gave up after %d attempts: %w", attempt, err)
+ break
+ }
+ slog.InfoContext(ctx, "Failed to initialize database", "attempt", attempt, "retryIn", wait, "error", err)
+ select {
+ case <-time.After(wait):
+ case <-ctx.Done():
+ return nil, fmt.Errorf("failed to connect to database: %w", ctx.Err())
}
}
if err != nil {
@@ -161,16 +165,15 @@ func Initialize(ctx context.Context, databaseURL string, options MigrationOption
if err != nil {
return nil, fmt.Errorf("failed to register database instrumentation: %w", err)
}
+ sqlDB.SetMaxIdleConns(kit.Ternary(dbProvider == dbProviderPostgres, 15, 5))
+ sqlDB.SetMaxOpenConns(kit.Ternary(dbProvider == dbProviderPostgres, 50, 20))
+ sqlDB.SetConnMaxLifetime(5 * time.Minute)
+ sqlDB.SetConnMaxIdleTime(3 * time.Minute)
metrics.ReportDBStatsMetrics(sqlDB)
if migrationErr := migrateDatabase(ctx, sqlDB, dbProvider, options, latestMigrationVersion); migrationErr != nil {
slog.ErrorContext(ctx, "Failed to run migrations", "error", migrationErr)
return nil, fmt.Errorf("failed to run migrations: %w", migrationErr)
}
-
- sqlDB.SetMaxIdleConns(kit.Ternary(dbProvider == dbProviderPostgres, 15, 5))
- sqlDB.SetMaxOpenConns(kit.Ternary(dbProvider == dbProviderPostgres, 50, 20))
- sqlDB.SetConnMaxLifetime(5 * time.Minute)
- sqlDB.SetConnMaxIdleTime(3 * time.Minute)
return db, nil
}
@@ -452,8 +455,16 @@ func ParseSQLiteConnectionString(connString string) (string, error) {
qs := sqliteutil.NormalizeQuery(connStringUrl.Query())
// Foreign-key enforcement is required on every pooled connection.
qs.Add("_pragma", "foreign_keys(1)")
- connStringUrl.RawQuery = qs.Encode()
- return connStringUrl.String(), nil
+ return (&url.URL{
+ Scheme: connStringUrl.Scheme,
+ Opaque: connStringUrl.Opaque,
+ User: connStringUrl.User,
+ Host: connStringUrl.Host,
+ Path: connStringUrl.Path,
+ RawPath: connStringUrl.RawPath,
+ RawQuery: qs.Encode(),
+ Fragment: connStringUrl.Fragment,
+ }).String(), nil
}
// FindEnvironmentIDByApiKey finds the environment ID that is associated with the given API key.
diff --git a/backend/internal/di/di.go b/backend/internal/di/di.go
index 776c1e6d0..69f4a67a3 100644
--- a/backend/internal/di/di.go
+++ b/backend/internal/di/di.go
@@ -4,6 +4,7 @@ package di
import (
"context"
"log/slog"
+ "net/http"
"go.uber.org/fx"
@@ -14,16 +15,20 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/build"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/dashboard"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/diagnostics"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
"github.com/getarcaneapp/arcane/backend/v2/internal/federated"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/health"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/job"
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/network"
@@ -32,7 +37,9 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/passkey"
"github.com/getarcaneapp/arcane/backend/v2/internal/port"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
+ s3domain "github.com/getarcaneapp/arcane/backend/v2/internal/s3"
"github.com/getarcaneapp/arcane/backend/v2/internal/search"
"github.com/getarcaneapp/arcane/backend/v2/internal/session"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
@@ -43,6 +50,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/upload"
"github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/variable"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/version"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
"github.com/getarcaneapp/arcane/backend/v2/internal/webhook"
@@ -53,10 +61,10 @@ import (
// ActorOptions provides the durable Francis runtime and admission.
var ActorOptions = fx.Options(
- fx.Provide(provideActorRuntimeInternal, provideRunCoordinatorInternal),
- fx.Provide(provideAdmissionGateInternal),
- fx.Provide(provideWorkflowEngineInternal),
- fx.Provide(provideTunnelRegistryInternal),
+ fx.Provide(provideActorRuntime, provideRunCoordinator),
+ fx.Provide(provideAdmissionGate),
+ fx.Provide(provideWorkflowEngine),
+ fx.Provide(provideTunnelRegistry),
)
// ServiceOptions constructs services once and injects them into route modules.
@@ -66,7 +74,7 @@ var ServiceOptions = fx.Options(
),
fx.Provide(
fx.Annotate(activity.NewActivityService, fx.As(fx.Self()), fx.As(new(activitylib.Service))),
- event.NewEventService,
+ fx.Annotate(event.NewEventService, fx.OnStop(func(ctx context.Context, service *event.EventService) error { return service.Stop(ctx) })),
job.NewJobService,
role.NewRoleService,
apikey.NewApiKeyService,
@@ -81,9 +89,9 @@ var ServiceOptions = fx.Options(
environment.NewEnvironmentService,
apns.NewApnsService,
notification.NewNotificationService,
- provideVulnerabilityServiceInternal,
- provideImageUpdateServiceInternal,
- provideImageServiceInternal,
+ provideVulnerabilityService,
+ provideImageUpdateService,
+ provideImageService,
build.NewBuildService,
project.NewLifecycleService,
container.NewContainerService,
@@ -93,16 +101,17 @@ var ServiceOptions = fx.Options(
swarm.NewSwarmService,
template.NewTemplateService,
oidc.NewOidcService,
- provideSystemServiceInternal,
+ provideSystemService,
diagnostics.NewDiagnosticsService,
- provideGitOpsSyncServiceInternal,
+ provideGitOpsSyncService,
variable.NewVariableService,
backup.NewRecoveryKeyStore,
upload.NewUploadService,
+ health.NewService,
auth.NewAuthService,
settings.NewSettingsSearchService,
fx.Annotate(settings.NewSettingsService, fx.OnStop(func(ctx context.Context, service *settings.SettingsService) error { return service.Stop(ctx) })),
- provideVolumeServiceInternal,
+ provideVolumeService,
fx.Annotate(webhook.NewWebhookService,
fx.OnStart(func(ctx context.Context, service *webhook.WebhookService) error { return service.LoadTokenHashes(ctx) }),
fx.OnStop(func(ctx context.Context, service *webhook.WebhookService) {
@@ -114,16 +123,45 @@ var ServiceOptions = fx.Options(
),
// These providers bind config values, callbacks, or managed resource lifetimes.
fx.Provide(
- provideJWKSetManagerInternal,
- provideDockerClientServiceInternal,
- provideProjectServiceInternal,
- provideBackupEngineInternal,
- provideVersionServiceInternal,
- provideGitRepositoryServiceInternal,
- provideS3ServiceInternal,
- provideContainerRegistryServiceInternal,
- provideUpdaterServiceInternal,
- provideAuthMiddlewareInternal,
+ provideJWKSetManager,
+ provideDockerClientService,
+ func(
+ db *database.DB,
+ localSettings *settings.SettingsService,
+ localEvent *event.EventService,
+ localImage *image.ImageService,
+ localDocker *docker.DockerClientService,
+ localBuild *build.BuildService,
+ lifecycleService *project.LifecycleService,
+ localKv *kv.KVService,
+ localRegistry *registry.ContainerRegistryService,
+ localEnvironment *environment.EnvironmentService,
+ cfg *config.Config,
+ ) *project.ProjectService {
+ return project.NewProjectService(db, localSettings, localEvent, localImage, localDocker, localBuild, lifecycleService, localRegistry, cfg, localKv, localEnvironment.GetEnabledRegistryCredentials)
+ },
+ provideBackupEngine,
+ func(
+ httpClient *http.Client,
+ cfg *config.Config,
+ localRegistry *registry.ContainerRegistryService,
+ localDocker *docker.DockerClientService,
+ imageUpdate *imageupdate.ImageUpdateService,
+ settingsService *settings.SettingsService,
+ ) *version.VersionService {
+ return version.NewVersionService(httpClient, cfg.UpdateCheckDisabled, config.Version, config.Revision, localRegistry, localDocker, imageUpdate, settingsService)
+ },
+ func(db *database.DB, cfg *config.Config, eventService *event.EventService, settingsService *settings.SettingsService) *gitrepo.GitRepositoryService {
+ return gitrepo.NewGitRepositoryService(db, cfg.GitWorkDir, eventService, settingsService)
+ },
+ func(db *database.DB, localEnvironment *environment.EnvironmentService) *s3domain.S3DestinationService {
+ return s3domain.NewS3DestinationService(db, localEnvironment.CheckS3DestinationReferences)
+ },
+ func(db *database.DB, localDocker *docker.DockerClientService, kvService *kv.KVService, settingsService *settings.SettingsService) *registry.ContainerRegistryService {
+ return registry.NewContainerRegistryService(db, func(ctx context.Context) (registry.RegistryDaemonClient, error) { return localDocker.GetClient(ctx) }, kvService, settingsService)
+ },
+ provideUpdaterService,
+ provideAuthMiddleware,
),
fx.Provide(
event.New,
@@ -149,13 +187,34 @@ var ServiceOptions = fx.Options(
variable.New,
upload.New,
search.New,
- provideActivityModuleInternal,
- provideImageUpdateModuleInternal,
- provideSettingsModuleInternal,
- provideS3ModuleInternal,
- provideContainerRegistryModuleInternal,
- provideAuthModuleInternal,
- provideUserModuleInternal,
+ health.New,
+ func(service *activity.ActivityService, localEnvironment *environment.EnvironmentService) *activity.Module {
+ return activity.New(service, activity.EnvironmentDependencies{
+ ProxyJSONRequest: localEnvironment.ProxyJSONRequest,
+ ListActiveRemoteEnvironments: localEnvironment.ListActiveRemoteEnvironments,
+ GetActiveRemoteEnvironment: localEnvironment.GetActiveRemoteEnvironmentSnapshot,
+ ProxyJSONRequestForEnvironment: localEnvironment.ProxyJSONRequestForEnvironment,
+ ResolveEnvironmentName: localEnvironment.ResolveEnvironmentName,
+ })
+ },
+ func(service *imageupdate.ImageUpdateService, imageService *image.ImageService) *imageupdate.Module {
+ return imageupdate.New(service, imageService.GetUpdateInfoByImageRefs)
+ },
+ func(service *settings.SettingsService, search *settings.SettingsSearchService, localEnvironment *environment.EnvironmentService, cfg *config.Config) *settings.Module {
+ return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)
+ },
+ func(service *s3domain.S3DestinationService, localEnvironment *environment.EnvironmentService) *s3domain.Module {
+ return s3domain.New(service, localEnvironment.SyncS3DestinationsToRemoteEnvironments)
+ },
+ func(service *registry.ContainerRegistryService, localEnvironment *environment.EnvironmentService) *registry.Module {
+ return registry.New(service, localEnvironment.SyncRegistriesToRemoteEnvironments)
+ },
+ func(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {
+ return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)
+ },
+ func(service *user.UserService, localAuth *auth.AuthService, settingsService *settings.SettingsService) *user.Module {
+ return user.New(service, localAuth.InvalidateUserTokenCache, settingsService)
+ },
),
)
@@ -171,7 +230,7 @@ var JobOptions = fx.Options(
scheduler.NewPruningVolumeHelperJob,
scheduler.NewExpiredSessionsCleanupJob,
scheduler.NewScheduledPruneJob,
- provideFilesystemWatcherJobInternal,
+ provideFilesystemWatcherJob,
fx.Annotate(scheduler.NewVulnerabilityScanJob, fx.ResultTags(`name:"vulnerability-scan"`)),
scheduler.NewVulnerabilityRiskJob,
fx.Annotate(scheduler.NewAutoPatchJob, fx.ResultTags(`name:"auto-patch"`)),
diff --git a/backend/internal/di/di_test.go b/backend/internal/di/di_test.go
index 0da1a084e..778136061 100644
--- a/backend/internal/di/di_test.go
+++ b/backend/internal/di/di_test.go
@@ -143,7 +143,7 @@ func TestOptionsValidate(t *testing.T) {
require.NoError(t, err)
}
-func TestDockerEventLifecycleInternal(t *testing.T) {
+func TestDockerEventLifecycle(t *testing.T) {
for _, mode := range []string{"manager", "agent", "edge"} {
t.Run(mode, func(t *testing.T) {
streamStopped := make(chan struct{})
@@ -188,7 +188,7 @@ func TestDockerEventLifecycleInternal(t *testing.T) {
cfg := &config.Config{DockerHost: server.URL, AgentMode: mode != "manager", EdgeAgent: mode == "edge"}
lifecycle := fxtest.NewLifecycle(t)
eventService := event.NewEventService(databaseDB, cfg, nil)
- provideDockerClientServiceInternal(t.Context(), lifecycle, databaseDB, cfg, nil, eventService)
+ provideDockerClientService(t.Context(), lifecycle, databaseDB, cfg, nil, eventService)
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 3*time.Second)
defer cancel()
diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.go
index 46e3fe233..7c0e390d6 100644
--- a/backend/internal/di/providers.go
+++ b/backend/internal/di/providers.go
@@ -18,7 +18,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
- "github.com/getarcaneapp/arcane/backend/v2/internal/build"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -32,7 +31,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
"github.com/getarcaneapp/arcane/backend/v2/internal/network"
"github.com/getarcaneapp/arcane/backend/v2/internal/notification"
- "github.com/getarcaneapp/arcane/backend/v2/internal/passkey"
"github.com/getarcaneapp/arcane/backend/v2/internal/project"
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
"github.com/getarcaneapp/arcane/backend/v2/internal/role"
@@ -42,7 +40,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/system"
"github.com/getarcaneapp/arcane/backend/v2/internal/template"
"github.com/getarcaneapp/arcane/backend/v2/internal/updater"
- "github.com/getarcaneapp/arcane/backend/v2/internal/user"
"github.com/getarcaneapp/arcane/backend/v2/internal/version"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
@@ -56,25 +53,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/oidcjwk"
)
-func provideActivityModuleInternal(service *activity.ActivityService, localEnvironment *environment.EnvironmentService) *activity.Module {
- return activity.New(service, activity.EnvironmentDependencies{
- ProxyJSONRequest: localEnvironment.ProxyJSONRequest,
- ListActiveRemoteEnvironments: localEnvironment.ListActiveRemoteEnvironments,
- GetActiveRemoteEnvironment: localEnvironment.GetActiveRemoteEnvironmentSnapshot,
- ProxyJSONRequestForEnvironment: localEnvironment.ProxyJSONRequestForEnvironment,
- ResolveEnvironmentName: localEnvironment.ResolveEnvironmentName,
- })
-}
-
-func provideImageUpdateModuleInternal(service *imageupdate.ImageUpdateService, imageService *image.ImageService) *imageupdate.Module {
- return imageupdate.New(service, imageService.GetUpdateInfoByImageRefs)
-}
-
-func provideSettingsModuleInternal(service *settings.SettingsService, search *settings.SettingsSearchService, localEnvironment *environment.EnvironmentService, cfg *config.Config) *settings.Module {
- return settings.New(service, search, localEnvironment.ProxyJSONRequest, cfg)
-}
-
-func provideBackupEngineInternal(lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, roles *role.RoleService, cfg *config.Config) *backup.Engine {
+func provideBackupEngine(lc fx.Lifecycle, admission *runs.Admission, imageService *image.ImageService, roles *role.RoleService, cfg *config.Config) *backup.Engine {
engine := backup.NewEngine(admission, imageService)
engine.SetAuthorize(func(ctx context.Context, requester backuptypes.Requester) error {
if cfg.AgentMode && requester.UserID == "agent" && requester.APIKeyID == "" {
@@ -99,7 +78,7 @@ func provideBackupEngineInternal(lc fx.Lifecycle, admission *runs.Admission, ima
return engine
}
-func provideActorRuntimeInternal(
+func provideActorRuntime(
appCtx context.Context,
cfg *config.Config,
db *database.DB,
@@ -130,7 +109,7 @@ func provideActorRuntimeInternal(
return runtime, nil
}
-func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVService, cfg *config.Config) (*runs.Coordinator, error) {
+func provideRunCoordinator(runtime *francis.Runtime, store *kv.KVService, cfg *config.Config) (*runs.Coordinator, error) {
coordinator := runs.New(store, runtime.Service(), cfg.GetLocation())
if err := coordinator.Register(runtime); err != nil {
return nil, err
@@ -138,9 +117,9 @@ func provideRunCoordinatorInternal(runtime *francis.Runtime, store *kv.KVService
return coordinator, nil
}
-// provideWorkflowEngineInternal starts monitoring after the host is ready; its
+// provideWorkflowEngine starts monitoring after the host is ready; its
// hook follows the runtime's because the engine depends on it.
-func provideWorkflowEngineInternal(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities *activity.ActivityService, lc fx.Lifecycle) (*flow.Engine, error) {
+func provideWorkflowEngine(appCtx context.Context, runtime *francis.Runtime, coordinator *runs.Coordinator, activities *activity.ActivityService, lc fx.Lifecycle) (*flow.Engine, error) {
engine, err := flow.New(appCtx, runtime, coordinator, activities)
if err != nil {
return nil, err
@@ -149,7 +128,7 @@ func provideWorkflowEngineInternal(appCtx context.Context, runtime *francis.Runt
return engine, nil
}
-func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, error) {
+func provideAdmissionGate(runtime *francis.Runtime) (*runs.Admission, error) {
admission := runs.NewAdmission(runtime.Service(), uuid.New().String())
if err := admission.Register(runtime); err != nil {
return nil, err
@@ -157,7 +136,7 @@ func provideAdmissionGateInternal(runtime *francis.Runtime) (*runs.Admission, er
return admission, nil
}
-func provideTunnelRegistryInternal(lc fx.Lifecycle) *edge.TunnelRegistry {
+func provideTunnelRegistry(lc fx.Lifecycle) *edge.TunnelRegistry {
localRegistry := edge.NewTunnelRegistry()
edge.SetDefaultRegistry(localRegistry)
lc.Append(fx.Hook{OnStop: func(ctx context.Context) error {
@@ -167,7 +146,7 @@ func provideTunnelRegistryInternal(lc fx.Lifecycle) *edge.TunnelRegistry {
return localRegistry
}
-func provideDockerClientServiceInternal(
+func provideDockerClientService(
ctx context.Context,
lc fx.Lifecycle,
db *database.DB,
@@ -219,69 +198,12 @@ func provideDockerClientServiceInternal(
return service
}
-func provideVersionServiceInternal(
- httpClient *http.Client,
- cfg *config.Config,
- localRegistry *registry.ContainerRegistryService,
- localDocker *docker.DockerClientService,
- imageUpdate *imageupdate.ImageUpdateService,
- settingsService *settings.SettingsService,
-) *version.VersionService {
- return version.NewVersionService(httpClient, cfg.UpdateCheckDisabled, config.Version, config.Revision, localRegistry, localDocker, imageUpdate, settingsService)
-}
-
-func provideGitRepositoryServiceInternal(db *database.DB, cfg *config.Config, eventService *event.EventService, settingsService *settings.SettingsService) *gitrepo.GitRepositoryService {
- return gitrepo.NewGitRepositoryService(db, cfg.GitWorkDir, eventService, settingsService)
-}
-
-func provideS3ModuleInternal(service *s3domain.S3DestinationService, localEnvironment *environment.EnvironmentService) *s3domain.Module {
- return s3domain.New(service, localEnvironment.SyncS3DestinationsToRemoteEnvironments)
-}
-
-func provideS3ServiceInternal(db *database.DB, localEnvironment *environment.EnvironmentService) *s3domain.S3DestinationService {
- return s3domain.NewS3DestinationService(db, localEnvironment.CheckS3DestinationReferences)
-}
-
-func provideAuthModuleInternal(service *auth.AuthService, userService *user.UserService, settingsService *settings.SettingsService, passkeyService *passkey.PasskeyService) *auth.Module {
- return auth.New(service, userService, settingsService, passkeyService.BeginMFAAuthentication)
-}
-
-func provideContainerRegistryServiceInternal(
- db *database.DB,
- localDocker *docker.DockerClientService,
- kvService *kv.KVService,
- settingsService *settings.SettingsService,
-) *registry.ContainerRegistryService {
- return registry.NewContainerRegistryService(db, func(ctx context.Context) (registry.RegistryDaemonClient, error) { return localDocker.GetClient(ctx) }, kvService, settingsService)
-}
-
-func provideContainerRegistryModuleInternal(service *registry.ContainerRegistryService, localEnvironment *environment.EnvironmentService) *registry.Module {
- return registry.New(service, localEnvironment.SyncRegistriesToRemoteEnvironments)
-}
-
-func provideProjectServiceInternal(
- db *database.DB,
- localSettings *settings.SettingsService,
- localEvent *event.EventService,
- localImage *image.ImageService,
- localDocker *docker.DockerClientService,
- localBuild *build.BuildService,
- lifecycleService *project.LifecycleService,
- localKv *kv.KVService,
- localRegistry *registry.ContainerRegistryService,
- localEnvironment *environment.EnvironmentService,
- cfg *config.Config,
-) *project.ProjectService {
- return project.NewProjectService(db, localSettings, localEvent, localImage, localDocker, localBuild, lifecycleService, localRegistry, cfg, localKv, localEnvironment.GetEnabledRegistryCredentials)
-}
-
// updaterServiceParams includes actor registration and worker lifecycle dependencies.
type updaterServiceParams struct {
fx.In
Config *config.Config
Workflows *flow.Engine
- Coordinator *runs.Coordinator
Admission *runs.Admission
Roles *role.RoleService
DB *database.DB
@@ -297,8 +219,8 @@ type updaterServiceParams struct {
Activity *activity.ActivityService
}
-// provideImageUpdateServiceInternal registers the image-check workflow while the host is still unstarted.
-func provideImageUpdateServiceInternal(
+// provideImageUpdateService registers the image-check workflow while the host is still unstarted.
+func provideImageUpdateService(
db *database.DB,
settingsService *settings.SettingsService,
registryService *registry.ContainerRegistryService,
@@ -315,8 +237,8 @@ func provideImageUpdateServiceInternal(
return service, nil
}
-// provideImageServiceInternal registers the auto-patch workflow while the host is still unstarted.
-func provideImageServiceInternal(
+// provideImageService registers the auto-patch workflow while the host is still unstarted.
+func provideImageService(
db *database.DB,
dockerService *docker.DockerClientService,
registryService *registry.ContainerRegistryService,
@@ -334,8 +256,8 @@ func provideImageServiceInternal(
return service, nil
}
-// provideVulnerabilityServiceInternal registers the scheduled scan workflow while the host is still unstarted.
-func provideVulnerabilityServiceInternal(
+// provideVulnerabilityService registers the scheduled scan workflow while the host is still unstarted.
+func provideVulnerabilityService(
db *database.DB,
cfg *config.Config,
dockerService *docker.DockerClientService,
@@ -355,7 +277,7 @@ func provideVulnerabilityServiceInternal(
return service, nil
}
-func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterService, error) {
+func provideUpdaterService(p updaterServiceParams) (*updater.UpdaterService, error) {
service, err := updater.NewUpdaterService(
p.DB,
p.Settings,
@@ -369,7 +291,6 @@ func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterServ
p.System,
p.Activity,
p.Config,
- p.Coordinator,
p.Admission,
p.Roles,
)
@@ -382,17 +303,13 @@ func provideUpdaterServiceInternal(p updaterServiceParams) (*updater.UpdaterServ
return service, nil
}
-func provideUserModuleInternal(service *user.UserService, localAuth *auth.AuthService, settingsService *settings.SettingsService) *user.Module {
- return user.New(service, localAuth.InvalidateUserTokenCache, settingsService)
-}
-
-func provideJWKSetManagerInternal(ctx context.Context, lc fx.Lifecycle) *oidcjwk.KeySetManager {
+func provideJWKSetManager(ctx context.Context, lc fx.Lifecycle) *oidcjwk.KeySetManager {
manager := oidcjwk.NewKeySetManager(ctx)
lc.Append(fx.Hook{OnStop: manager.Shutdown})
return manager
}
-func provideAuthMiddlewareInternal(
+func provideAuthMiddleware(
authService *auth.AuthService,
apiKey *apikey.ApiKeyService,
env *environment.EnvironmentService,
@@ -405,7 +322,7 @@ func provideAuthMiddlewareInternal(
WithPermissionResolver(localRole)
}
-func provideFilesystemWatcherJobInternal(
+func provideFilesystemWatcherJob(
ctx context.Context,
lc fx.Lifecycle,
localProject *project.ProjectService,
@@ -461,8 +378,8 @@ func provideFilesystemWatcherJobInternal(
return job, nil
}
-// provideGitOpsSyncServiceInternal registers the gitops-sync workflow while the host is still unstarted.
-func provideGitOpsSyncServiceInternal(
+// provideGitOpsSyncService registers the gitops-sync workflow while the host is still unstarted.
+func provideGitOpsSyncService(
db *database.DB,
repoService *gitrepo.GitRepositoryService,
projectService *project.ProjectService,
@@ -478,8 +395,8 @@ func provideGitOpsSyncServiceInternal(
return service, nil
}
-// provideVolumeServiceInternal registers the volume backup workflows while the host is still unstarted.
-func provideVolumeServiceInternal(
+// provideVolumeService registers the volume backup workflows while the host is still unstarted.
+func provideVolumeService(
lc fx.Lifecycle,
db *database.DB,
dockerService *docker.DockerClientService,
@@ -501,8 +418,8 @@ func provideVolumeServiceInternal(
return service, nil
}
-// provideSystemServiceInternal registers the system backup workflows while the host is still unstarted.
-func provideSystemServiceInternal(
+// provideSystemService registers the system backup workflows while the host is still unstarted.
+func provideSystemService(
db *database.DB,
dockerService *docker.DockerClientService,
containerService *container.ContainerService,
diff --git a/backend/internal/docker/service.go b/backend/internal/docker/service.go
index c3cecacfc..e15ccb2d1 100644
--- a/backend/internal/docker/service.go
+++ b/backend/internal/docker/service.go
@@ -77,30 +77,6 @@ func (s *DockerClientService) WithClient(dockerClient *client.Client) *DockerCli
}
}
-func detectDockerAPIVersion(ctx context.Context, host string) (string, error) {
- probeClient, err := client.New(client.WithHost(host))
- if err != nil {
- return "", fmt.Errorf("failed to create Docker probe client: %w", err)
- }
- defer closeDockerClient(ctx, probeClient, "probe")
-
- ctx, cancel := context.WithTimeout(ctx, dockerClientNegotiationTimeout)
- defer cancel()
-
- pingResult, err := probeClient.Ping(ctx, client.PingOptions{})
- if err != nil {
- return "", fmt.Errorf("failed to negotiate Docker API version: %w", err)
- }
-
- apiVersion := strings.TrimSpace(pingResult.APIVersion)
- if apiVersion == "" {
- slog.WarnContext(ctx, "Docker ping did not report an API version, using minimum supported client API version", "apiVersion", client.MinAPIVersion)
- return client.MinAPIVersion, nil
- }
-
- return apiVersion, nil
-}
-
func closeDockerClient(ctx context.Context, cli *client.Client, purpose string) {
if cli == nil {
return
@@ -115,44 +91,40 @@ func closeDockerClient(ctx context.Context, cli *client.Client, purpose string)
// It initializes the client on the first call.
func (s *DockerClientService) GetClient(ctx context.Context) (*client.Client, error) {
s.mu.Lock()
- if s.Client != nil {
- cli := s.Client
- s.mu.Unlock()
+ cli := s.Client
+ s.mu.Unlock()
+ if cli != nil {
return cli, nil
}
- s.mu.Unlock()
- apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)
- if err != nil {
- return nil, fmt.Errorf("failed to create Docker client: %w", err)
- }
- cli, err := client.New(client.WithHost(s.config.DockerHost), client.WithAPIVersion(apiVersion))
- if err != nil {
- return nil, fmt.Errorf("failed to create Docker client: failed to configure Docker client API version %s: %w", apiVersion, err)
+ if err := s.RefreshClient(ctx); err != nil {
+ return nil, err
}
s.mu.Lock()
- if s.Client != nil {
- existingClient := s.Client
- s.mu.Unlock()
- closeDockerClient(ctx, cli, "unused after concurrent initialization")
- return existingClient, nil
- }
-
- s.Client = cli
- s.clientVersion = cli.ClientVersion()
- s.clientLastProbe = time.Now()
- s.mu.Unlock()
-
- return cli, nil
+ defer s.mu.Unlock()
+ return s.Client, nil
}
// RefreshClient probes the Docker daemon and recreates the cached client when
// the daemon's effective API version changed.
func (s *DockerClientService) RefreshClient(ctx context.Context) error {
- apiVersion, err := detectDockerAPIVersion(ctx, s.config.DockerHost)
+ probeClient, err := client.New(client.WithHost(s.config.DockerHost))
if err != nil {
- return fmt.Errorf("failed to refresh Docker client: %w", err)
+ return fmt.Errorf("failed to refresh Docker client: failed to create Docker probe client: %w", err)
+ }
+ pingCtx, cancel := context.WithTimeout(ctx, dockerClientNegotiationTimeout)
+ pingResult, err := probeClient.Ping(pingCtx, client.PingOptions{})
+ cancel()
+ closeDockerClient(ctx, probeClient, "probe")
+ if err != nil {
+ return fmt.Errorf("failed to refresh Docker client: failed to negotiate Docker API version: %w", err)
+ }
+
+ apiVersion := strings.TrimSpace(pingResult.APIVersion)
+ if apiVersion == "" {
+ slog.WarnContext(ctx, "Docker ping did not report an API version, using minimum supported client API version", "apiVersion", client.MinAPIVersion)
+ apiVersion = client.MinAPIVersion
}
s.mu.Lock()
@@ -219,11 +191,15 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {
eventBackoff.InitialInterval = 500 * time.Millisecond
eventBackoff.MaxInterval = 30 * time.Second
+ // Resume from the last received event so a reconnect replays events missed during the gap.
+ var lastEventNano int64
for ctx.Err() == nil {
if dockerClient, err := s.GetClient(ctx); err != nil {
slog.WarnContext(ctx, "failed to connect to Docker event stream", "error", err)
} else {
- result := dockerClient.Events(ctx, client.EventsListOptions{})
+ result := dockerClient.Events(ctx, client.EventsListOptions{
+ Since: kit.Ternary(lastEventNano == 0, "", fmt.Sprintf("%d.%09d", lastEventNano/int64(time.Second), lastEventNano%int64(time.Second))),
+ })
streamStart := time.Now()
var streamErr error
stream:
@@ -239,6 +215,7 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {
if msg.TimeNano == 0 {
msg.TimeNano = time.Now().UnixNano()
}
+ lastEventNano = msg.TimeNano
s.EventBus().Publish(msg)
case recvErr, ok := <-result.Err:
streamErr = kit.Ternary(ok, recvErr, nil)
@@ -261,16 +238,21 @@ func (s *DockerClientService) WatchEvents(ctx context.Context) {
return
case <-timer.C:
}
+ // A restarted daemon may report a different API version, so re-probe before reconnecting.
+ if err := s.RefreshClient(ctx); err != nil {
+ slog.DebugContext(ctx, "failed to refresh Docker client before event stream reconnect", "error", err)
+ }
}
}
-// listCoalesced shares one detached op among concurrent callers; each waiter stops on its own
+// ListCoalesced shares one detached Call among concurrent callers; each waiter stops on its own
// context, nothing is cached, and the shared result must be treated as immutable.
-func listCoalesced[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {
+// Streaming calls (logs, events, attach, pull, exec, stats, wait) must not use it because Call applies a deadline.
+func (s *DockerClientService) ListCoalesced[T any](ctx context.Context, group *singleflight.Group, op func(context.Context, *client.Client) (T, error)) (T, error) {
var zero T
ch := group.DoChan("list", func() (any, error) {
- return op(context.WithoutCancel(ctx))
+ return s.Call(context.WithoutCancel(ctx), op)
})
select {
@@ -288,8 +270,9 @@ func listCoalesced[T any](ctx context.Context, group *singleflight.Group, op fun
}
}
-// callAPI runs op against the cached client under the configured Docker API timeout.
-func callAPI[T any](ctx context.Context, s *DockerClientService, op func(context.Context, *client.Client) (T, error)) (T, error) {
+// Call runs op against the cached client under the configured Docker API timeout.
+// Streaming calls (logs, events, attach, pull, exec, stats, wait) must not use it because it applies a deadline.
+func (s *DockerClientService) Call[T any](ctx context.Context, op func(context.Context, *client.Client) (T, error)) (T, error) {
dockerClient, err := s.GetClient(ctx)
if err != nil {
var zero T
@@ -305,36 +288,22 @@ func callAPI[T any](ctx context.Context, s *DockerClientService, op func(context
}
func (s *DockerClientService) ListContainers(ctx context.Context) ([]container.Summary, error) {
- return listCoalesced(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {
- return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
- containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
- if err != nil {
- return nil, fmt.Errorf("failed to list Docker containers: %w", err)
- }
- return containerList.Items, nil
- })
+ return s.ListCoalesced(ctx, &s.containerListFlight, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
+ containerList, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ if err != nil {
+ return nil, fmt.Errorf("failed to list Docker containers: %w", err)
+ }
+ return containerList.Items, nil
})
}
func (s *DockerClientService) ListImages(ctx context.Context) ([]image.Summary, error) {
- return listCoalesced(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {
- return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]image.Summary, error) {
- imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{All: true})
- if err != nil {
- return nil, fmt.Errorf("failed to list Docker images: %w", err)
- }
- return imageList.Items, nil
- })
- })
-}
-
-func (s *DockerClientService) listNetworks(ctx context.Context) ([]network.Summary, error) {
- return callAPI(ctx, s, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {
- networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ return s.ListCoalesced(ctx, &s.imageListFlight, func(ctx context.Context, dockerClient *client.Client) ([]image.Summary, error) {
+ imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{All: true})
if err != nil {
- return nil, fmt.Errorf("failed to list Docker networks: %w", err)
+ return nil, fmt.Errorf("failed to list Docker images: %w", err)
}
- return networkList.Items, nil
+ return imageList.Items, nil
})
}
@@ -364,7 +333,13 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*d
defer utils.RecoverToError(&workerErr, "docker snapshot worker")
var err error
- networks, err = s.listNetworks(groupCtx)
+ networks, err = s.Call(groupCtx, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {
+ networkList, listErr := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ if listErr != nil {
+ return nil, fmt.Errorf("failed to list Docker networks: %w", listErr)
+ }
+ return networkList.Items, nil
+ })
if err != nil {
slog.WarnContext(groupCtx, "failed to list Docker networks for snapshot", "error", err)
}
@@ -374,7 +349,7 @@ func (s *DockerClientService) GetSnapshot(ctx context.Context, envID string) (*d
defer utils.RecoverToError(&workerErr, "docker snapshot worker")
var err error
- volumes, err = callAPI(groupCtx, s, func(ctx context.Context, dockerClient *client.Client) (*client.VolumeListResult, error) {
+ volumes, err = s.Call(groupCtx, func(ctx context.Context, dockerClient *client.Client) (*client.VolumeListResult, error) {
volResp, listErr := dockerClient.VolumeList(ctx, client.VolumeListOptions{})
if listErr != nil {
return nil, fmt.Errorf("failed to list Docker volumes: %w", listErr)
@@ -476,7 +451,13 @@ func (s *DockerClientService) GetAllNetworks(ctx context.Context) ([]network.Sum
if err != nil {
return nil, 0, 0, 0, err
}
- networks, err := s.listNetworks(ctx)
+ networks, err := s.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]network.Summary, error) {
+ networkList, listErr := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ if listErr != nil {
+ return nil, fmt.Errorf("failed to list Docker networks: %w", listErr)
+ }
+ return networkList.Items, nil
+ })
if err != nil {
return nil, 0, 0, 0, err
}
diff --git a/backend/internal/docker/service_test.go b/backend/internal/docker/service_test.go
index e1e537ceb..fd803a156 100644
--- a/backend/internal/docker/service_test.go
+++ b/backend/internal/docker/service_test.go
@@ -274,6 +274,62 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStream(t *testing.T) {
}, time.Second, time.Millisecond)
}
+func TestDockerClientService_WatchEventsResumesFromLastEventAfterRefresh(t *testing.T) {
+ first := events.Message{
+ Type: events.ImageEventType,
+ Action: events.ActionPull,
+ TimeNano: 1_700_000_000_123_456_789,
+ }
+ payload, err := json.Marshal(first)
+ require.NoError(t, err)
+ var pings, eventCalls atomic.Int32
+ type reconnect struct {
+ since string
+ pings int32
+ }
+ reconnected := make(chan reconnect, 1)
+ server := newDockerPingTestServerWithHandler(t, func(w http.ResponseWriter, r *http.Request) {
+ switch path.Base(r.URL.Path) {
+ case "_ping":
+ pings.Add(1)
+ w.Header().Set("Api-Version", "1.41")
+ w.WriteHeader(http.StatusOK)
+ case "events":
+ w.Header().Set("Content-Type", "application/json")
+ if eventCalls.Add(1) == 1 {
+ assert.Empty(t, r.URL.Query().Get("since"))
+ _, _ = w.Write(payload)
+ return
+ }
+ select {
+ case reconnected <- reconnect{since: r.URL.Query().Get("since"), pings: pings.Load()}:
+ default:
+ }
+ w.WriteHeader(http.StatusOK)
+ <-r.Context().Done()
+ default:
+ http.NotFound(w, r)
+ }
+ })
+ svc := newDockerClientServiceForTest(t, server.URL)
+ t.Cleanup(svc.Close)
+ stop, err := concurrency.StartSupervised(t.Context(), "Docker event watcher", func(ctx context.Context) error {
+ svc.WatchEvents(ctx)
+ return nil
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = stop(context.WithoutCancel(t.Context())) })
+
+ select {
+ case got := <-reconnected:
+ assert.Equal(t, "1700000000.123456789", got.since)
+ // One probe from the initial GetClient, one from the RefreshClient before reconnecting.
+ assert.GreaterOrEqual(t, got.pings, int32(2))
+ case <-time.After(5 * time.Second):
+ t.Fatal("event stream did not reconnect")
+ }
+}
+
func TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {
images := []image.Summary{
{ID: "sha256:image-a", Containers: -1, Size: 100},
diff --git a/backend/internal/environment/children/health/service.go b/backend/internal/environment/children/health/service.go
index 59e4914b8..931c108e2 100644
--- a/backend/internal/environment/children/health/service.go
+++ b/backend/internal/environment/children/health/service.go
@@ -34,11 +34,9 @@ func (s *Service) Probe(ctx context.Context, id, apiURL string, isEdge, customUR
reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return "offline", fmt.Errorf("failed to connect to Docker: %w", err)
- }
- if _, pingErr := dockerClient.Ping(reqCtx, client.PingOptions{}); pingErr != nil {
+ if _, pingErr := s.dockerService.Call(reqCtx, func(ctx context.Context, cli *client.Client) (client.PingResult, error) {
+ return cli.Ping(ctx, client.PingOptions{})
+ }); pingErr != nil {
return "offline", fmt.Errorf("docker ping failed: %w", pingErr)
}
return "online", nil
diff --git a/backend/internal/event/service.go b/backend/internal/event/service.go
index bae2265e0..aa4cc725e 100644
--- a/backend/internal/event/service.go
+++ b/backend/internal/event/service.go
@@ -18,8 +18,6 @@ import (
"github.com/getarcaneapp/arcane/types/v2/event"
"github.com/moby/moby/api/types/events"
- "github.com/samber/mo"
- "github.com/samber/mo/option"
"go.getarcane.app/kit/pkg"
"go.getarcane.app/kit/pkg/mapping"
"go.getarcane.app/streams/agg"
@@ -43,6 +41,12 @@ import (
// maxConcurrentEdgeSyncs bounds in-flight event forwards to the manager.
const maxConcurrentEdgeSyncs = 16
+const (
+ daemonEventQueueSize = 1024
+ daemonEventBatchSize = 64
+ daemonEventFlushInterval = 250 * time.Millisecond
+)
+
type EventService struct {
changes *concurrency.Signal[struct{}]
correlation *correlation.Service
@@ -50,6 +54,12 @@ type EventService struct {
cfg *config.Config
httpClient *http.Client
edgeSyncs *semaphore.Weighted
+ // Daemon events are persisted in batches by a writer started on first use.
+ daemonQueue chan CreateEventRequest
+ daemonStart sync.Once
+ daemonDone chan struct{}
+ daemonMu sync.RWMutex
+ daemonStopped bool
}
func NewEventService(db *database.DB, cfg *config.Config, httpClient *http.Client) *EventService {
@@ -65,6 +75,8 @@ func NewEventService(db *database.DB, cfg *config.Config, httpClient *http.Clien
cfg: cfg,
httpClient: httpClient,
edgeSyncs: semaphore.NewWeighted(maxConcurrentEdgeSyncs),
+ daemonQueue: make(chan CreateEventRequest, daemonEventQueueSize),
+ daemonDone: make(chan struct{}),
}
}
@@ -85,53 +97,62 @@ type CreateEventRequest struct {
}
func (s *EventService) CreateEvent(ctx context.Context, req CreateEventRequest) (*Event, error) {
- severity := cmp.Or(req.Severity, EventSeverityInfo)
- userID, username := normalizeEventActor(req.UserID, req.Username)
-
- eventRecord := &Event{
- CreatedAt: time.Now(),
- Type: req.Type,
- Severity: severity,
- Title: req.Title,
- Description: req.Description,
- ResourceType: req.ResourceType,
- ResourceID: req.ResourceID,
- ResourceName: req.ResourceName,
- UserID: userID,
- Username: username,
- EnvironmentID: req.EnvironmentID,
- Metadata: req.Metadata,
- Timestamp: time.Now(),
- }
-
- if req.deduplicationKey != "" {
- eventRecord.DeduplicationKey = &req.deduplicationKey
+ records, err := s.persistEvents(ctx, []CreateEventRequest{req})
+ if err != nil {
+ return nil, err
}
+ return records[0], nil
+}
- var inserted bool
+// persistEvents inserts requests in one transaction, skipping replayed deduplication keys.
+func (s *EventService) persistEvents(ctx context.Context, reqs []CreateEventRequest) ([]*Event, error) {
+ records := make([]*Event, len(reqs))
+ inserted := make([]bool, len(reqs))
err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if req.deduplicationKey != "" {
- tx = tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "deduplication_key"}}, DoNothing: true})
- }
- result := tx.Create(eventRecord)
- if result.Error != nil {
- return fmt.Errorf("failed to create event: %w", result.Error)
+ for index, req := range reqs {
+ // A missing actor half mirrors the other; no actor at all is the system.
+ userID, username := strings.TrimSpace(kit.FromPtr(req.UserID)), strings.TrimSpace(kit.FromPtr(req.Username))
+ records[index] = &Event{
+ CreatedAt: time.Now(),
+ Type: req.Type,
+ Severity: cmp.Or(req.Severity, EventSeverityInfo),
+ Title: req.Title,
+ Description: req.Description,
+ ResourceType: req.ResourceType,
+ ResourceID: req.ResourceID,
+ ResourceName: req.ResourceName,
+ UserID: new(cmp.Or(userID, username, "system")),
+ Username: new(cmp.Or(username, userID, "System")),
+ EnvironmentID: req.EnvironmentID,
+ Metadata: req.Metadata,
+ Timestamp: time.Now(),
+ DeduplicationKey: kit.Ternary(req.deduplicationKey != "", &req.deduplicationKey, nil),
+ }
+ create := tx
+ if req.deduplicationKey != "" {
+ create = tx.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "deduplication_key"}}, DoNothing: true})
+ }
+ result := create.Create(records[index])
+ if result.Error != nil {
+ return fmt.Errorf("failed to create event: %w", result.Error)
+ }
+ inserted[index] = result.RowsAffected > 0
}
- inserted = result.RowsAffected > 0
return nil
})
if err != nil {
return nil, err
}
- if !inserted {
- return eventRecord, nil
+ for index, req := range reqs {
+ if !inserted[index] {
+ continue
+ }
+ s.correlateCreatedEvent(req)
+ s.changes.Publish(struct{}{})
+ s.forwardEventToManager(ctx, records[index])
}
- s.correlateCreatedEvent(req)
- s.changes.Publish(struct{}{})
- s.forwardEventToManager(ctx, eventRecord)
-
- return eventRecord, nil
+ return records, nil
}
// IngestAgentEvent assigns the authenticated environment instead of trusting the agent's local ID.
@@ -145,38 +166,30 @@ func (s *EventService) IngestAgentEvent(ctx context.Context, environmentID strin
}
func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Event) {
- if eventModel == nil || s.cfg == nil || !s.cfg.AgentMode {
+ if s.cfg == nil || !s.cfg.AgentMode {
return
}
evt := &edge.TunnelEvent{
- Type: string(eventModel.Type),
- Severity: string(eventModel.Severity),
- Title: eventModel.Title,
- Description: eventModel.Description,
- }
- if eventModel.ResourceType != nil {
- evt.ResourceType = *eventModel.ResourceType
- }
- if eventModel.ResourceID != nil {
- evt.ResourceID = *eventModel.ResourceID
- }
- if eventModel.ResourceName != nil {
- evt.ResourceName = *eventModel.ResourceName
- }
- if eventModel.UserID != nil {
- evt.UserID = *eventModel.UserID
- }
- if eventModel.Username != nil {
- evt.Username = *eventModel.Username
- }
- if eventModel.Metadata != nil {
- metadataBytes, err := json.Marshal(map[string]any(eventModel.Metadata))
- if err != nil {
- slog.WarnContext(ctx, "Failed to marshal event metadata for edge sync", "type", eventModel.Type, "error", err)
- } else {
- evt.MetadataJSON = metadataBytes
- }
+ Type: string(eventModel.Type),
+ Severity: string(eventModel.Severity),
+ Title: eventModel.Title,
+ Description: eventModel.Description,
+ ResourceType: kit.FromPtr(eventModel.ResourceType),
+ ResourceID: kit.FromPtr(eventModel.ResourceID),
+ ResourceName: kit.FromPtr(eventModel.ResourceName),
+ UserID: kit.FromPtr(eventModel.UserID),
+ Username: kit.FromPtr(eventModel.Username),
+ MetadataJSON: func() []byte {
+ if eventModel.Metadata == nil {
+ return nil
+ }
+ metadataBytes, err := json.Marshal(map[string]any(eventModel.Metadata))
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to marshal event metadata for edge sync", "type", eventModel.Type, "error", err)
+ }
+ return metadataBytes
+ }(),
}
go func(parentCtx context.Context, outgoing *edge.TunnelEvent) {
@@ -194,7 +207,7 @@ func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Ev
slog.WarnContext(syncCtx, "Failed to sync event to manager over edge tunnel", "type", outgoing.Type, "error", err)
return
}
- if !s.canForwardEventToManagerHTTP() {
+ if strings.TrimSpace(s.cfg.AgentToken) == "" || strings.TrimSpace(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl)) == "" {
return
}
if httpErr := s.forwardEventToManagerHTTP(syncCtx, eventModel); httpErr != nil {
@@ -205,30 +218,19 @@ func (s *EventService) forwardEventToManager(ctx context.Context, eventModel *Ev
}(ctx, evt)
}
-func (s *EventService) canForwardEventToManagerHTTP() bool {
- if s.cfg == nil {
- return false
- }
- if strings.TrimSpace(s.cfg.AgentToken) == "" {
- return false
- }
- return strings.TrimSpace(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl)) != ""
-}
-
func (s *EventService) forwardEventToManagerHTTP(ctx context.Context, eventModel *Event) error {
- if eventModel == nil {
- return errors.New("event is required")
+ baseURL, err := url.Parse(strings.TrimSpace(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl)))
+ if err != nil {
+ return fmt.Errorf("manager API URL is invalid for manager event sync: failed to parse manager API URL: %w", err)
}
- if s.cfg == nil || strings.TrimSpace(s.cfg.AgentToken) == "" {
- return errors.New("agent token is required for manager event sync")
+ if baseURL.Scheme != "http" && baseURL.Scheme != "https" {
+ return fmt.Errorf("manager API URL is invalid for manager event sync: unsupported scheme %q", baseURL.Scheme)
}
-
- managerEventsURL, err := managerEventEndpointURL(httpx.ManagerBaseURL(s.cfg.ManagerApiUrl))
- if err != nil {
- return fmt.Errorf("manager API URL is invalid for manager event sync: %w", err)
+ if baseURL.Host == "" {
+ return errors.New("manager API URL is invalid for manager event sync: manager API URL host is required")
}
- payload := CreateEventRequest{
+ body, err := json.Marshal(CreateEventRequest{
Type: eventModel.Type,
Severity: eventModel.Severity,
Title: eventModel.Title,
@@ -239,18 +241,18 @@ func (s *EventService) forwardEventToManagerHTTP(ctx context.Context, eventModel
UserID: eventModel.UserID,
Username: eventModel.Username,
EnvironmentID: eventModel.EnvironmentID,
- }
-
- if len(eventModel.Metadata) > 0 {
- payload.Metadata = eventModel.Metadata
- }
-
- body, err := json.Marshal(payload)
+ Metadata: eventModel.Metadata,
+ })
if err != nil {
return fmt.Errorf("failed to marshal event payload: %w", err)
}
- req, err := http.NewRequestWithContext(ctx, http.MethodPost, managerEventsURL, bytes.NewReader(body))
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, (&url.URL{
+ Scheme: baseURL.Scheme,
+ User: baseURL.User,
+ Host: baseURL.Host,
+ Path: strings.TrimRight(baseURL.Path, "/") + "/api/events",
+ }).String(), bytes.NewReader(body))
if err != nil {
return fmt.Errorf("failed to create manager event request: %w", err)
}
@@ -274,62 +276,16 @@ func (s *EventService) forwardEventToManagerHTTP(ctx context.Context, eventModel
return fmt.Errorf("manager event sync failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
-func managerEventEndpointURL(rawBaseURL string) (string, error) {
- trimmed := strings.TrimSpace(rawBaseURL)
- if trimmed == "" {
- return "", errors.New("manager API URL is required")
- }
-
- baseURL, err := url.Parse(trimmed)
- if err != nil {
- return "", fmt.Errorf("failed to parse manager API URL: %w", err)
- }
- if baseURL.Scheme != "http" && baseURL.Scheme != "https" {
- return "", fmt.Errorf("unsupported scheme %q", baseURL.Scheme)
- }
- if baseURL.Host == "" {
- return "", errors.New("manager API URL host is required")
- }
-
- baseURL.RawQuery = ""
- baseURL.Fragment = ""
- baseURL.Path = strings.TrimRight(baseURL.Path, "/") + "/api/events"
- return baseURL.String(), nil
-}
-
-func normalizeEventActor(userID, username *string) (*string, *string) {
- normalizedUserID := normalizeOptionalStringPtr(userID)
- normalizedUsername := normalizeOptionalStringPtr(username)
-
- if normalizedUsername == nil && normalizedUserID != nil {
- normalizedUsername = new(*normalizedUserID)
- }
- if normalizedUserID == nil && normalizedUsername != nil {
- normalizedUserID = new(*normalizedUsername)
- }
- if normalizedUserID == nil && normalizedUsername == nil {
- normalizedUserID = new("system")
- normalizedUsername = new("System")
- }
-
- return normalizedUserID, normalizedUsername
+func (s *EventService) ListEventsPaginated(ctx context.Context, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {
+ return paginateEvents(pagination.ApplyFilter(s.db.WithContext(ctx).Model(&Event{}), "environment_id", params.Filters["environmentId"]), params)
}
-func normalizeOptionalStringPtr(value *string) *string {
- if value == nil {
- return nil
- }
- trimmed := strings.TrimSpace(*value)
- if trimmed == "" {
- return nil
- }
- return &trimmed
+func (s *EventService) GetEventsByEnvironmentPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {
+ return paginateEvents(s.db.WithContext(ctx).Model(&Event{}).Where("environment_id = ?", environmentID), params)
}
-func (s *EventService) ListEventsPaginated(ctx context.Context, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {
- var eventRecords []Event
- q := s.db.WithContext(ctx).Model(&Event{})
-
+// paginateEvents applies the shared search and filters to q, then pages and maps the rows.
+func paginateEvents(q *gorm.DB, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {
if term := strings.TrimSpace(params.Search); term != "" {
searchPattern := "%" + term + "%"
q = q.Where(
@@ -338,42 +294,34 @@ func (s *EventService) ListEventsPaginated(ctx context.Context, params paginatio
)
}
- q = pagination.ApplyFilter(q, "severity", params.Filters["severity"])
- q = applyEventTypeFilter(q, params.Filters["type"])
- q = pagination.ApplyFilter(q, "resource_type", params.Filters["resourceType"])
- q = pagination.ApplyFilter(q, "username", params.Filters["username"])
- q = pagination.ApplyFilter(q, "environment_id", params.Filters["environmentId"])
-
- paginationResp, err := pagination.PaginateAndSortDB(params, q, &eventRecords)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to paginate events: %w", err)
+ // Type values with a '.' are exact types; others are category prefixes. Comma-separated values are OR-ed.
+ var typeConds []string
+ var typeArgs []any
+ var exactTypes []string
+ for part := range strings.SplitSeq(params.Filters["type"], ",") {
+ part = strings.TrimSpace(part)
+ switch {
+ case part == "":
+ case strings.Contains(part, "."):
+ exactTypes = append(exactTypes, part)
+ default:
+ typeConds = append(typeConds, "type LIKE ?")
+ typeArgs = append(typeArgs, part+".%")
+ }
}
-
- eventDtos, mapErr := mapping.MapSlice[Event, event.Event](eventRecords)
- if mapErr != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to map events: %w", mapErr)
+ if len(exactTypes) > 0 {
+ typeConds = append(typeConds, "type IN ?")
+ typeArgs = append(typeArgs, exactTypes)
}
-
- return eventDtos, paginationResp, nil
-}
-
-func (s *EventService) GetEventsByEnvironmentPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]event.Event, pagination.Response, error) {
- var eventRecords []Event
- q := s.db.WithContext(ctx).Model(&Event{}).Where("environment_id = ?", environmentID)
-
- if term := strings.TrimSpace(params.Search); term != "" {
- searchPattern := "%" + term + "%"
- q = q.Where(
- "title LIKE ? OR description LIKE ? OR COALESCE(resource_name, '') LIKE ? OR COALESCE(username, '') LIKE ?",
- searchPattern, searchPattern, searchPattern, searchPattern,
- )
+ if len(typeConds) > 0 {
+ q = q.Where(strings.Join(typeConds, " OR "), typeArgs...)
}
q = pagination.ApplyFilter(q, "severity", params.Filters["severity"])
- q = applyEventTypeFilter(q, params.Filters["type"])
q = pagination.ApplyFilter(q, "resource_type", params.Filters["resourceType"])
q = pagination.ApplyFilter(q, "username", params.Filters["username"])
+ var eventRecords []Event
paginationResp, err := pagination.PaginateAndSortDB(params, q, &eventRecords)
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to paginate events: %w", err)
@@ -387,41 +335,6 @@ func (s *EventService) GetEventsByEnvironmentPaginated(ctx context.Context, envi
return eventDtos, paginationResp, nil
}
-// applyEventTypeFilter filters by event type. Values containing a '.' are
-// exact types (e.g. "container.start"); values without one are category
-// prefixes (e.g. "container" matches "container.%"). Comma-separated values
-// are OR-ed together, mirroring pagination.ApplyFilter's multi-value handling.
-func applyEventTypeFilter(q *gorm.DB, value string) *gorm.DB {
- if value == "" {
- return q
- }
- var (
- exact []string
- conds []string
- args []any
- )
- for part := range strings.SplitSeq(value, ",") {
- part = strings.TrimSpace(part)
- if part == "" {
- continue
- }
- if strings.Contains(part, ".") {
- exact = append(exact, part)
- } else {
- conds = append(conds, "type LIKE ?")
- args = append(args, part+".%")
- }
- }
- if len(exact) > 0 {
- conds = append(conds, "type IN ?")
- args = append(args, exact)
- }
- if len(conds) == 0 {
- return q
- }
- return q.Where(strings.Join(conds, " OR "), args...)
-}
-
func (s *EventService) GetEventSeverityCounts(ctx context.Context) (event.SeverityCounts, error) {
var rows []struct {
Severity string
@@ -489,60 +402,28 @@ func (s *EventService) DeleteOldEvents(ctx context.Context, olderThan time.Durat
}
func (s *EventService) LogContainerEvent(ctx context.Context, eventType EventType, containerID, containerName, userID, username, environmentID string, metadata database.JSON) error {
- title := s.generateEventTitle(eventType, containerName)
- description := s.generateEventDescription(eventType, "container", containerName)
- severity := s.getEventSeverity(eventType)
-
- _, err := s.CreateEvent(ctx, CreateEventRequest{
- Type: eventType,
- Severity: severity,
- Title: title,
- Description: description,
- ResourceType: new("container"),
- ResourceID: new(containerID),
- ResourceName: new(containerName),
- UserID: new(userID),
- Username: new(username),
- EnvironmentID: new(environmentID),
- Metadata: metadata,
- })
- return err
+ return s.logResourceEvent(ctx, eventType, "container", containerID, containerName, userID, username, environmentID, metadata)
}
func (s *EventService) LogImageEvent(ctx context.Context, eventType EventType, imageID, imageName, userID, username, environmentID string, metadata database.JSON) error {
- title := s.generateEventTitle(eventType, imageName)
- description := s.generateEventDescription(eventType, "image", imageName)
- severity := s.getEventSeverity(eventType)
-
- _, err := s.CreateEvent(ctx, CreateEventRequest{
- Type: eventType,
- Severity: severity,
- Title: title,
- Description: description,
- ResourceType: new("image"),
- ResourceID: new(imageID),
- ResourceName: new(imageName),
- UserID: new(userID),
- Username: new(username),
- EnvironmentID: new(environmentID),
- Metadata: metadata,
- })
- return err
+ return s.logResourceEvent(ctx, eventType, "image", imageID, imageName, userID, username, environmentID, metadata)
}
func (s *EventService) LogProjectEvent(ctx context.Context, eventType EventType, projectID, projectName, userID, username, environmentID string, metadata database.JSON) error {
- title := s.generateEventTitle(eventType, projectName)
- description := s.generateEventDescription(eventType, "project", projectName)
- severity := s.getEventSeverity(eventType)
+ return s.logResourceEvent(ctx, eventType, "project", projectID, projectName, userID, username, environmentID, metadata)
+}
+// logResourceEvent records a successful operation on a resource using its eventDefinitions entry.
+func (s *EventService) logResourceEvent(ctx context.Context, eventType EventType, resourceType, resourceID, resourceName, userID, username, environmentID string, metadata database.JSON) error {
+ title, description, severity := describeEvent(eventType, resourceType, resourceName)
_, err := s.CreateEvent(ctx, CreateEventRequest{
Type: eventType,
Severity: severity,
Title: title,
Description: description,
- ResourceType: new("project"),
- ResourceID: new(projectID),
- ResourceName: new(projectName),
+ ResourceType: new(resourceType),
+ ResourceID: new(resourceID),
+ ResourceName: new(resourceName),
UserID: new(userID),
Username: new(username),
EnvironmentID: new(environmentID),
@@ -552,10 +433,7 @@ func (s *EventService) LogProjectEvent(ctx context.Context, eventType EventType,
}
func (s *EventService) LogUserEvent(ctx context.Context, eventType EventType, userID, username string, metadata database.JSON) error {
- title := s.generateEventTitle(eventType, username)
- description := s.generateEventDescription(eventType, "user", username)
- severity := s.getEventSeverity(eventType)
-
+ title, description, severity := describeEvent(eventType, "user", username)
_, err := s.CreateEvent(ctx, CreateEventRequest{
Type: eventType,
Severity: severity,
@@ -569,45 +447,11 @@ func (s *EventService) LogUserEvent(ctx context.Context, eventType EventType, us
}
func (s *EventService) LogVolumeEvent(ctx context.Context, eventType EventType, volumeID, volumeName, userID, username, environmentID string, metadata database.JSON) error {
- title := s.generateEventTitle(eventType, volumeName)
- description := s.generateEventDescription(eventType, "volume", volumeName)
- severity := s.getEventSeverity(eventType)
-
- _, err := s.CreateEvent(ctx, CreateEventRequest{
- Type: eventType,
- Severity: severity,
- Title: title,
- Description: description,
- ResourceType: new("volume"),
- ResourceID: new(volumeID),
- ResourceName: new(volumeName),
- UserID: new(userID),
- Username: new(username),
- EnvironmentID: new(environmentID),
- Metadata: metadata,
- })
- return err
+ return s.logResourceEvent(ctx, eventType, "volume", volumeID, volumeName, userID, username, environmentID, metadata)
}
func (s *EventService) LogNetworkEvent(ctx context.Context, eventType EventType, networkID, networkName, userID, username, environmentID string, metadata database.JSON) error {
- title := s.generateEventTitle(eventType, networkName)
- description := s.generateEventDescription(eventType, "network", networkName)
- severity := s.getEventSeverity(eventType)
-
- _, err := s.CreateEvent(ctx, CreateEventRequest{
- Type: eventType,
- Severity: severity,
- Title: title,
- Description: description,
- ResourceType: new("network"),
- ResourceID: new(networkID),
- ResourceName: new(networkName),
- UserID: new(userID),
- Username: new(username),
- EnvironmentID: new(environmentID),
- Metadata: metadata,
- })
- return err
+ return s.logResourceEvent(ctx, eventType, "network", networkID, networkName, userID, username, environmentID, metadata)
}
func (s *EventService) LogErrorEvent(ctx context.Context, eventType EventType, resourceType, resourceID, resourceName, userID, username, environmentID string, err error, metadata database.JSON) {
@@ -737,27 +581,13 @@ var eventDefinitions = map[EventType]eventDefinition{
EventTypeFederatedExchange: {"Federated credential exchange: %s", "Federated credential exchange for '%s'", EventSeverityInfo},
}
-func (s *EventService) generateEventTitle(eventType EventType, resourceName string) string {
- definition, ok := eventDefinitions[eventType]
- return option.Map(func(def eventDefinition) string {
- return fmt.Sprintf(def.TitleFormat, resourceName)
- })(mo.TupleToOption(definition, ok)).OrElse("Event: " + string(eventType))
-}
-
-func (s *EventService) generateEventDescription(eventType EventType, resourceType, resourceName string) string {
- definition, ok := eventDefinitions[eventType]
- return option.Map(func(def eventDefinition) string {
- return fmt.Sprintf(def.DescriptionFormat, resourceName)
- })(mo.TupleToOption(definition, ok)).OrElse(
- fmt.Sprintf("%s operation performed on %s '%s'", string(eventType), resourceType, resourceName),
- )
-}
-
-func (s *EventService) getEventSeverity(eventType EventType) EventSeverity {
+// describeEvent formats an event's title, description, and severity, with a generic fallback for undefined types.
+func describeEvent(eventType EventType, resourceType, resourceName string) (string, string, EventSeverity) {
definition, ok := eventDefinitions[eventType]
- return option.Map(func(def eventDefinition) EventSeverity {
- return def.Severity
- })(mo.TupleToOption(definition, ok)).OrElse(EventSeverityInfo)
+ if !ok {
+ return "Event: " + string(eventType), fmt.Sprintf("%s operation performed on %s '%s'", string(eventType), resourceType, resourceName), EventSeverityInfo
+ }
+ return fmt.Sprintf(definition.TitleFormat, resourceName), fmt.Sprintf(definition.DescriptionFormat, resourceName), definition.Severity
}
// RunStreamProducer signals committed event changes so clients can refresh their current query.
@@ -866,11 +696,77 @@ func (s *EventService) RecordDockerEvent(ctx context.Context, msg events.Message
}
req.deduplicationKey = fmt.Sprintf("%x", sha256.Sum256(identity))
}
+ s.daemonStart.Do(func() { go s.writeDaemonEvents(context.WithoutCancel(ctx)) })
+ s.daemonMu.RLock()
+ queued := false
+ if !s.daemonStopped {
+ select {
+ case s.daemonQueue <- req:
+ queued = true
+ default:
+ }
+ }
+ s.daemonMu.RUnlock()
+ if queued {
+ return
+ }
+ // A full queue or stopped writer falls back to a direct insert rather than dropping the event.
if _, err := s.CreateEvent(ctx, req); err != nil {
slog.WarnContext(ctx, "Failed to log Docker daemon event", "type", req.Type, "resourceId", msg.Actor.ID, "error", err)
}
}
+func (s *EventService) writeDaemonEvents(ctx context.Context) {
+ defer close(s.daemonDone)
+ batch := make([]CreateEventRequest, 0, daemonEventBatchSize)
+ var flush <-chan time.Time
+ for open := true; open; {
+ select {
+ case req, ok := <-s.daemonQueue:
+ open = ok
+ if ok {
+ batch = append(batch, req)
+ if flush == nil {
+ flush = time.After(daemonEventFlushInterval)
+ }
+ if len(batch) < daemonEventBatchSize {
+ continue
+ }
+ }
+ case <-flush:
+ }
+ if _, err := s.persistEvents(ctx, batch); err != nil {
+ slog.WarnContext(ctx, "Failed to persist Docker daemon event batch; retrying individually", "count", len(batch), "error", err)
+ for _, req := range batch {
+ if _, createErr := s.CreateEvent(ctx, req); createErr != nil {
+ slog.WarnContext(ctx, "Failed to log Docker daemon event", "type", req.Type, "resourceId", kit.FromPtr(req.ResourceID), "error", createErr)
+ }
+ }
+ }
+ batch = batch[:0]
+ flush = nil
+ }
+}
+
+// Stop flushes queued daemon events; later observations are persisted directly.
+func (s *EventService) Stop(ctx context.Context) error {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ s.daemonMu.Lock()
+ if !s.daemonStopped {
+ s.daemonStopped = true
+ close(s.daemonQueue)
+ }
+ s.daemonMu.Unlock()
+ s.daemonStart.Do(func() { close(s.daemonDone) })
+ select {
+ case <-s.daemonDone:
+ return nil
+ case <-ctx.Done():
+ return fmt.Errorf("failed to flush Docker daemon events: %w", ctx.Err())
+ }
+}
+
var daemonEventTypes = map[events.Type]map[events.Action]EventType{
events.ContainerEventType: {
events.ActionCreate: EventTypeContainerCreate,
@@ -992,7 +888,7 @@ func (s *EventService) correlateCreatedEvent(req CreateEventRequest) {
if req.ResourceType == nil || req.Metadata["source"] == "docker" {
return
}
- if req.EnvironmentID != nil && *req.EnvironmentID != "" && *req.EnvironmentID != "0" {
+ if cmp.Or(kit.FromPtr(req.EnvironmentID), "0") != "0" {
return
}
// Failure and observational records do not identify a successful Docker mutation.
@@ -1007,12 +903,5 @@ func (s *EventService) correlateCreatedEvent(req CreateEventRequest) {
default:
return
}
- var id, name string
- if req.ResourceID != nil {
- id = *req.ResourceID
- }
- if req.ResourceName != nil {
- name = *req.ResourceName
- }
- s.MarkDockerExpectation(*req.ResourceType, id, name)
+ s.correlation.MarkExpectation(*req.ResourceType, kit.FromPtr(req.ResourceID), kit.FromPtr(req.ResourceName))
}
diff --git a/backend/internal/event/service_test.go b/backend/internal/event/service_test.go
index 085e2a51a..a3a2e6f26 100644
--- a/backend/internal/event/service_test.go
+++ b/backend/internal/event/service_test.go
@@ -7,6 +7,7 @@ import (
"net/http"
"net/http/httptest"
"strings"
+ "sync"
"sync/atomic"
"testing"
"testing/synctest"
@@ -439,6 +440,7 @@ func TestDaemonEventForwardingPreservesMetadata(t *testing.T) {
require.True(t, ok)
svc.RecordDockerEvent(t.Context(), message)
svc.RecordDockerEvent(t.Context(), message)
+ require.NoError(t, svc.Stop(t.Context()))
select {
case payload := <-requests:
require.Equal(t, req.Metadata, payload.Metadata)
@@ -707,7 +709,7 @@ func TestMapDaemonEvent(t *testing.T) {
}
}
-func TestDaemonEventsPersistEveryOccurrenceImmediately(t *testing.T) {
+func TestDaemonEventsPersistEveryOccurrence(t *testing.T) {
db := setupEventServiceTestDB(t)
svc := NewEventService(db, nil, nil)
die := events.Message{Type: events.ContainerEventType, Action: events.ActionDie, Actor: events.Actor{ID: "web", Attributes: map[string]string{"exitCode": "0"}}}
@@ -716,8 +718,9 @@ func TestDaemonEventsPersistEveryOccurrenceImmediately(t *testing.T) {
for index, msg := range []events.Message{die, stop, die, stop} {
msg.TimeNano = int64(index + 1)
svc.RecordDockerEvent(t.Context(), msg)
- requireDaemonCount(t, db, int64(index+1))
}
+ require.NoError(t, svc.Stop(t.Context()))
+ requireDaemonCount(t, db, 4)
var rows []Event
require.NoError(t, db.Find(&rows).Error)
for _, row := range rows {
@@ -746,6 +749,7 @@ func TestDaemonSuppressionAndFailureAllowLaterObservations(t *testing.T) {
svc.MarkDockerExpectation("container", "web", "")
}
svc.RecordDockerEvent(t.Context(), msg)
+ require.NoError(t, svc.Stop(t.Context()))
requireDaemonCount(t, db, 0)
if failure {
require.NoError(t, db.Callback().Create().Remove("daemon_failure"))
@@ -798,10 +802,14 @@ func TestDockerEventSubscriptionsReadyAndStop(t *testing.T) {
done := make(chan error, 1)
go func() { done <- run(ctx) }()
synctest.Wait()
+ time.Sleep(daemonEventFlushInterval)
+ synctest.Wait()
requireDaemonCount(t, db, 4*occurrences)
require.Equal(t, int64(4*occurrences), published.Load())
publishBurst(occurrences)
synctest.Wait()
+ time.Sleep(daemonEventFlushInterval)
+ synctest.Wait()
requireDaemonCount(t, db, 8*occurrences)
require.Equal(t, int64(8*occurrences), published.Load())
if closeBus {
@@ -811,6 +819,7 @@ func TestDockerEventSubscriptionsReadyAndStop(t *testing.T) {
}
synctest.Wait()
require.NoError(t, <-done)
+ require.NoError(t, svc.Stop(t.Context()))
cleanup()
cleanup()
})
@@ -818,6 +827,49 @@ func TestDockerEventSubscriptionsReadyAndStop(t *testing.T) {
}
}
+func TestDaemonEventWriterBatchesAndFlushesOnStop(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ db := setupEventServiceTestDB(t)
+ sqlDB, err := db.DB.DB()
+ require.NoError(t, err)
+ defer func() { assert.NoError(t, sqlDB.Close()) }()
+ sqlDB.SetMaxOpenConns(1)
+ svc := NewEventService(db, nil, nil)
+ defer func() { assert.NoError(t, svc.Stop(t.Context())) }()
+ var published atomic.Int64
+ stopChanges := svc.changes.Subscribe(func(struct{}) { published.Add(1) })
+ defer stopChanges()
+ record := func(timeNano int64) {
+ svc.RecordDockerEvent(t.Context(), events.Message{Type: events.ContainerEventType, Action: events.ActionStart, TimeNano: timeNano, Actor: events.Actor{ID: "web"}})
+ }
+
+ const occurrences = daemonEventBatchSize*3 + 5
+ var wg sync.WaitGroup
+ for occurrence := range int64(occurrences) {
+ for range 2 {
+ wg.Go(func() { record(occurrence + 1) })
+ }
+ }
+ wg.Wait()
+ synctest.Wait()
+ time.Sleep(daemonEventFlushInterval)
+ synctest.Wait()
+ requireDaemonCount(t, db, occurrences)
+ require.Equal(t, int64(occurrences), published.Load())
+
+ // Fewer than a batch stays queued until the flush timer, which Stop preempts.
+ for occurrence := range int64(5) {
+ record(occurrences + occurrence + 1)
+ }
+ require.NoError(t, svc.Stop(t.Context()))
+ requireDaemonCount(t, db, occurrences+5)
+
+ record(occurrences + 6)
+ requireDaemonCount(t, db, occurrences+6)
+ require.Equal(t, int64(occurrences+6), published.Load())
+ })
+}
+
func requireDaemonCount(t *testing.T, db *database.DB, want int64) {
t.Helper()
var count int64
diff --git a/backend/internal/gitops/service.go b/backend/internal/gitops/service.go
index e36e58eeb..636d1cbec 100644
--- a/backend/internal/gitops/service.go
+++ b/backend/internal/gitops/service.go
@@ -46,6 +46,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)
@@ -827,7 +828,7 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, id
return nil, err
}
- status := &gitops.SyncStatus{
+ return &gitops.SyncStatus{
ID: syncRecord.ID,
AutoSync: syncRecord.AutoSync,
LastSyncAt: syncRecord.LastSyncAt,
@@ -839,14 +840,13 @@ func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, id
BackupPending: syncRecord.BackupPending,
BackupFailureReason: syncRecord.BackupFailureReason,
LastBackupAt: syncRecord.LastBackupAt,
- }
-
- // Calculate next sync time
- if syncRecord.AutoSync && syncRecord.LastSyncAt != nil {
- status.NextSyncAt = new(syncRecord.LastSyncAt.Add(time.Duration(syncRecord.SyncInterval) * time.Minute))
- }
-
- return status, nil
+ NextSyncAt: func() *time.Time {
+ if !syncRecord.AutoSync || syncRecord.LastSyncAt == nil {
+ return nil
+ }
+ return new(syncRecord.LastSyncAt.Add(time.Duration(syncRecord.SyncInterval) * time.Minute))
+ }(),
+ }, nil
}
// CleanupLeakedScratchDirsOnStartup removes GitOps scratch directories an interrupted sync left in the projects
@@ -1063,6 +1063,7 @@ func (r *backupRuntime) schedule(syncID string, run func()) {
timer.Stop()
}
r.timers[syncID] = time.AfterFunc(r.debounce, func() {
+ defer utils.RecoverToError(nil, "git backup debounce", "syncId", syncID)
r.mu.Lock()
delete(r.timers, syncID)
r.mu.Unlock()
@@ -1122,6 +1123,7 @@ func (s *GitOpsSyncService) SubscribeProjectFileChanges(ctx context.Context) {
return
}
go func() {
+ defer utils.RecoverToError(nil, "git backup after save", "syncId", syncID, "projectId", projectID)
if _, err := s.PerformSync(runCtx, environmentID, syncID, user.SystemUser); err != nil {
slog.ErrorContext(runCtx, "git backup after save failed", "syncId", syncID, "error", err)
}
diff --git a/backend/internal/health/handler.go b/backend/internal/health/handler.go
index 0e8045e5e..84014e4f5 100644
--- a/backend/internal/health/handler.go
+++ b/backend/internal/health/handler.go
@@ -5,12 +5,11 @@ import (
"net/http"
"github.com/getarcaneapp/arcane/types/v2/system"
-
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+ kit "go.getarcane.app/kit/pkg"
)
type healthHandler struct {
- actors *francis.Runtime
+ service *Service
}
type healthOutput struct {
@@ -22,6 +21,11 @@ type headHealthOutput struct {
Status int
}
+type readyOutput struct {
+ Status int
+ Body system.ReadinessResponse
+}
+
type healthOutputs interface {
healthOutput | headHealthOutput
}
@@ -30,20 +34,27 @@ type healthOutputs interface {
func health[T healthOutputs](h *healthHandler) func(context.Context, *struct{}) (*T, error) {
return func(context.Context, *struct{}) (*T, error) {
status, body := http.StatusOK, system.HealthResponse{Status: "UP"}
- if h.actors != nil {
- select {
- case <-h.actors.Done():
- status, body = http.StatusServiceUnavailable, system.HealthResponse{Status: "DOWN"}
- default:
- }
+ select {
+ case <-h.service.shutdown:
+ status, body = http.StatusServiceUnavailable, system.HealthResponse{Status: "DOWN"}
+ default:
}
var output T
switch typed := any(&output).(type) {
case *healthOutput:
- typed.Status, typed.Body = status, body
+ *typed = healthOutput{Status: status, Body: body}
case *headHealthOutput:
- typed.Status = status
+ *typed = headHealthOutput{Status: status}
}
return &output, nil
}
}
+
+// ready serves the readiness probe; 503 lists the failing check.
+func (h *healthHandler) ready(ctx context.Context, _ *struct{}) (*readyOutput, error) {
+ result := h.service.Ready(ctx)
+ return &readyOutput{
+ Status: kit.Ternary(result.Status == readyStatus, http.StatusOK, http.StatusServiceUnavailable),
+ Body: result,
+ }, nil
+}
diff --git a/backend/internal/health/module.go b/backend/internal/health/module.go
index 50fe624ef..fd216dcb3 100644
--- a/backend/internal/health/module.go
+++ b/backend/internal/health/module.go
@@ -4,13 +4,21 @@ import (
"net/http"
"github.com/danielgtaylor/huma/v2"
-
- "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
-// RegisterRoutes registers the health check routes.
-func RegisterRoutes(api huma.API, actors *francis.Runtime) {
- handler := &healthHandler{actors: actors}
+// Module owns the liveness and readiness routes.
+type Module struct {
+ service *Service
+}
+
+// New builds the health module that registers the liveness and readiness routes.
+func New(service *Service) *Module {
+ return &Module{service: service}
+}
+
+// RegisterRoutes registers the liveness and readiness routes; readiness is hidden from the OpenAPI spec.
+func (m *Module) RegisterRoutes(api huma.API) {
+ handler := &healthHandler{service: m.service}
huma.Register(api, huma.Operation{
OperationID: "health-check",
@@ -31,4 +39,15 @@ func RegisterRoutes(api huma.API, actors *francis.Runtime) {
Tags: []string{"Health"},
Security: []map[string][]string{},
}, health[headHealthOutput](handler))
+
+ huma.Register(api, huma.Operation{
+ OperationID: "health-ready",
+ Method: http.MethodGet,
+ Path: "/health/ready",
+ Summary: "Readiness probe",
+ Description: "Report whether the database and Docker daemon are reachable",
+ Tags: []string{"Health"},
+ Security: []map[string][]string{},
+ Hidden: true,
+ }, handler.ready)
}
diff --git a/backend/internal/health/service.go b/backend/internal/health/service.go
new file mode 100644
index 000000000..0612434d4
--- /dev/null
+++ b/backend/internal/health/service.go
@@ -0,0 +1,82 @@
+package health
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/system"
+ "github.com/moby/moby/client"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
+)
+
+const (
+ readyStatus = "ready"
+ notReadyStatus = "not_ready"
+ checkOK = "ok"
+ checkTimeout = 2 * time.Second
+)
+
+// Service checks the dependencies the manager needs before it can serve traffic.
+type Service struct {
+ db *database.DB
+ docker *docker.DockerClientService
+ shutdown <-chan struct{}
+}
+
+// NewService builds the readiness checker over the database, the local Docker client, and the actor runtime's shutdown signal.
+func NewService(db *database.DB, dockerService *docker.DockerClientService, actors *francis.Runtime) *Service {
+ return &Service{
+ db: db,
+ docker: dockerService,
+ shutdown: actors.Done(),
+ }
+}
+
+// Ready pings the database and local Docker daemon; it is never ready while shutting down.
+func (s *Service) Ready(ctx context.Context) system.ReadinessResponse {
+ select {
+ case <-s.shutdown:
+ return system.ReadinessResponse{
+ Status: notReadyStatus,
+ Checks: map[string]string{"shutdown": "in progress"},
+ }
+ default:
+ }
+
+ checks := map[string]string{
+ "database": func() string {
+ sqlDB, err := s.db.SQLDB()
+ if err != nil {
+ return fmt.Sprintf("get database handle: %v", err)
+ }
+ pingCtx, cancel := context.WithTimeout(ctx, checkTimeout)
+ defer cancel()
+ if pingErr := sqlDB.PingContext(pingCtx); pingErr != nil {
+ return pingErr.Error()
+ }
+ return checkOK
+ }(),
+ "docker": func() string {
+ pingCtx, cancel := context.WithTimeout(ctx, checkTimeout)
+ defer cancel()
+ dockerClient, err := s.docker.GetClient(pingCtx)
+ if err != nil {
+ return fmt.Sprintf("connect to Docker: %v", err)
+ }
+ if _, pingErr := dockerClient.Ping(pingCtx, client.PingOptions{}); pingErr != nil {
+ return pingErr.Error()
+ }
+ return checkOK
+ }(),
+ }
+ for _, result := range checks {
+ if result != checkOK {
+ return system.ReadinessResponse{Status: notReadyStatus, Checks: checks}
+ }
+ }
+ return system.ReadinessResponse{Status: readyStatus, Checks: checks}
+}
diff --git a/backend/internal/health/service_test.go b/backend/internal/health/service_test.go
new file mode 100644
index 000000000..aa267ab32
--- /dev/null
+++ b/backend/internal/health/service_test.go
@@ -0,0 +1,80 @@
+package health
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/getarcaneapp/arcane/types/v2/system"
+ "github.com/libtnb/sqlite"
+ "github.com/moby/moby/client"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+)
+
+func newReadyTestService(t *testing.T, shutdown <-chan struct{}) (*Service, *database.DB) {
+ t.Helper()
+
+ gormDB, err := gorm.Open(sqlite.Open("file:health-ready?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+ db := &database.DB{DB: gormDB}
+ t.Cleanup(func() { _ = db.Close() })
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Api-Version", "1.41")
+ _, _ = w.Write([]byte("OK"))
+ }))
+ t.Cleanup(server.Close)
+
+ dockerClient, err := client.New(
+ client.WithHost(server.URL),
+ client.WithAPIVersion("1.41"),
+ client.WithHTTPClient(server.Client()),
+ )
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = dockerClient.Close() })
+
+ return &Service{
+ db: db,
+ docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),
+ shutdown: shutdown,
+ }, db
+}
+
+func TestService_Ready(t *testing.T) {
+ t.Run("ready when database and docker respond", func(t *testing.T) {
+ svc, _ := newReadyTestService(t, make(chan struct{}))
+
+ assert.Equal(t, system.ReadinessResponse{
+ Status: readyStatus,
+ Checks: map[string]string{
+ "database": checkOK,
+ "docker": checkOK,
+ },
+ }, svc.Ready(t.Context()))
+ })
+
+ t.Run("not ready when database ping fails", func(t *testing.T) {
+ svc, db := newReadyTestService(t, make(chan struct{}))
+ require.NoError(t, db.Close())
+
+ result := svc.Ready(t.Context())
+ assert.Equal(t, notReadyStatus, result.Status)
+ assert.NotEqual(t, checkOK, result.Checks["database"])
+ assert.Equal(t, checkOK, result.Checks["docker"])
+ })
+
+ t.Run("not ready while shutting down", func(t *testing.T) {
+ shutdown := make(chan struct{})
+ close(shutdown)
+ svc, _ := newReadyTestService(t, shutdown)
+
+ result := svc.Ready(t.Context())
+ assert.Equal(t, notReadyStatus, result.Status)
+ assert.Equal(t, map[string]string{"shutdown": "in progress"}, result.Checks)
+ })
+}
diff --git a/backend/internal/image/children/attestations/service.go b/backend/internal/image/children/attestations/service.go
index e5b24e218..0454853ab 100644
--- a/backend/internal/image/children/attestations/service.go
+++ b/backend/internal/image/children/attestations/service.go
@@ -28,6 +28,7 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"go.getarcane.app/kit/pkg"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"
)
@@ -42,8 +43,8 @@ var zstdDecoder = sync.OnceValues(func() (*zstd.Decoder, error) { return zstd.Ne
// Service reads in-toto attestations attached to local images and registry references.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
- registryAuth func(ctx context.Context, registryHost string) (string, error)
+ dockerService *docker.DockerClientService
+ registryAuth func(ctx context.Context, registryHost string) (string, error)
}
type Query struct {
@@ -71,10 +72,10 @@ type attestationStatementEnvelope struct {
}
func NewService(
- dockerClient func(ctx context.Context) (*client.Client, error),
+ dockerService *docker.DockerClientService,
registryAuth func(ctx context.Context, registryHost string) (string, error),
) *Service {
- return &Service{dockerClient: dockerClient, registryAuth: registryAuth}
+ return &Service{dockerService: dockerService, registryAuth: registryAuth}
}
// GetImageAttestations returns in-toto attestations attached to a local image or registry reference.
@@ -198,28 +199,28 @@ func (s *Service) resolveImageAttestationReference(ctx context.Context, imageNam
return reference{}, errors.New("image name is required")
}
- if dockerClient, err := s.dockerClient(ctx); err == nil {
- if inspect, inspectErr := dockerClient.ImageInspect(ctx, imageName); inspectErr == nil {
- imageRef := func() string {
- for _, candidate := range slices.Concat(
- slices.DeleteFunc(slices.Clone(inspect.RepoDigests), func(repoDigest string) bool { return !strings.Contains(repoDigest, "@sha256:") }),
- inspect.RepoTags,
- ) {
- if trimmed := strings.TrimSpace(candidate); trimmed != "" && !strings.Contains(trimmed, "<none>") {
- return trimmed
- }
+ if inspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageName)
+ }); inspectErr == nil {
+ imageRef := func() string {
+ for _, candidate := range slices.Concat(
+ slices.DeleteFunc(slices.Clone(inspect.RepoDigests), func(repoDigest string) bool { return !strings.Contains(repoDigest, "@sha256:") }),
+ inspect.RepoTags,
+ ) {
+ if trimmed := strings.TrimSpace(candidate); trimmed != "" && !strings.Contains(trimmed, "<none>") {
+ return trimmed
}
- return ""
- }()
- if imageRef == "" {
- return reference{ImageRef: imageName, Empty: true}, nil
}
- _, subjectDigest, _ := strings.Cut(imageRef, "@")
- return reference{
- ImageRef: imageRef,
- SubjectDigest: subjectDigest,
- }, nil
+ return ""
+ }()
+ if imageRef == "" {
+ return reference{ImageRef: imageName, Empty: true}, nil
}
+ _, subjectDigest, _ := strings.Cut(imageRef, "@")
+ return reference{
+ ImageRef: imageRef,
+ SubjectDigest: subjectDigest,
+ }, nil
}
if strings.HasPrefix(imageName, "sha256:") || (len(imageName) == 64 && !strings.ContainsFunc(imageName, func(r rune) bool {
diff --git a/backend/internal/image/children/patch/service.go b/backend/internal/image/children/patch/service.go
index e2b4fc69a..57e1d57e6 100644
--- a/backend/internal/image/children/patch/service.go
+++ b/backend/internal/image/children/patch/service.go
@@ -172,14 +172,12 @@ func (s *Service) startPatch(
}
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
- if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {
+ if storeErr := s.requireContainerdImageStore(ctx); storeErr != nil {
return nil, nil, storeErr
}
- imageInspect, err := dockerClient.ImageInspect(ctx, imageID)
+ imageInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageID)
+ })
if err != nil {
return nil, nil, fmt.Errorf("failed to inspect image: %w", err)
}
@@ -435,15 +433,15 @@ func (s *Service) runPatch(ctx context.Context, record *ImagePatchRecord, opts i
// verifyPatchedImage warns when the expected tag is missing (copa may arch-suffix
// multi-platform tags) and otherwise re-scans it to show whether the patch worked.
func (s *Service) verifyPatchedImage(ctx context.Context, record *ImagePatchRecord, activityID string) {
- if dockerClient, err := s.dockerService.GetClient(ctx); err == nil {
- if patchedInspect, inspectErr := dockerClient.ImageInspect(ctx, record.PatchedRef); inspectErr != nil {
- slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", inspectErr)
- s.appendPatchActivity(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")
- } else if s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
- s.appendPatchActivity(ctx, activityID, 95, "Re-scanning patched image to verify the patch")
- if _, scanErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, scheduledPatchUser); scanErr != nil {
- slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanErr)
- }
+ if patchedInspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, record.PatchedRef)
+ }); inspectErr != nil {
+ slog.WarnContext(ctx, "patched image tag not found after patching", "patchedRef", record.PatchedRef, "error", inspectErr)
+ s.appendPatchActivity(ctx, activityID, 95, "Patched image was created but the expected tag "+record.PatchedRef+" was not found; check the image list")
+ } else if s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {
+ s.appendPatchActivity(ctx, activityID, 95, "Re-scanning patched image to verify the patch")
+ if _, scanErr := s.vulnerabilityService.ScanImage(context.WithoutCancel(ctx), record.EnvironmentID, patchedInspect.ID, scheduledPatchUser); scanErr != nil {
+ slog.WarnContext(ctx, "failed to start verification scan of patched image", "patchedRef", record.PatchedRef, "error", scanErr)
}
}
}
@@ -844,11 +842,7 @@ func (s *Service) Targets(ctx context.Context, envID string) ([]Target, error) {
if err := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); err != nil {
return nil, err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
- if storeErr := requireContainerdImageStore(ctx, dockerClient); storeErr != nil {
+ if storeErr := s.requireContainerdImageStore(ctx); storeErr != nil {
return nil, storeErr
}
var scans []vulnerability.VulnerabilityScanRecord
@@ -885,8 +879,10 @@ func (s *Service) Targets(ctx context.Context, envID string) ([]Target, error) {
}
// Copa needs BuildKit's docker exporter, which dockerd only offers with the containerd image store.
-func requireContainerdImageStore(ctx context.Context, dockerClient *client.Client) error {
- info, err := dockerClient.Info(ctx, client.InfoOptions{})
+func (s *Service) requireContainerdImageStore(ctx context.Context) error {
+ info, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SystemInfoResult, error) {
+ return dockerClient.Info(ctx, client.InfoOptions{})
+ })
if err != nil {
return fmt.Errorf("failed to inspect Docker: %w", err)
}
diff --git a/backend/internal/image/module.go b/backend/internal/image/module.go
index b6c2c5497..5ea746a5b 100644
--- a/backend/internal/image/module.go
+++ b/backend/internal/image/module.go
@@ -42,15 +42,14 @@ func New(
uploadService *upload.UploadService,
) *Module {
return &Module{
- service: service,
- docker: dockerService,
- imageUpdate: imageUpdate,
- settings: settingsService,
- build: buildService,
- activity: activityService,
- upload: uploadService,
- // The attestation reader uses the image service's Docker client and registry credentials.
- attestationsHandler: attestations.NewHandler(attestations.NewService(service.dockerService.GetClient, service.registryService.GetRegistryAuthForHost)),
+ service: service,
+ docker: dockerService,
+ imageUpdate: imageUpdate,
+ settings: settingsService,
+ build: buildService,
+ activity: activityService,
+ upload: uploadService,
+ attestationsHandler: attestations.NewHandler(attestations.NewService(service.dockerService, service.registryService.GetRegistryAuthForHost)),
}
}
diff --git a/backend/internal/image/service.go b/backend/internal/image/service.go
index 9980a94ed..a74c79679 100644
--- a/backend/internal/image/service.go
+++ b/backend/internal/image/service.go
@@ -47,6 +47,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/registryauth"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
@@ -225,11 +226,6 @@ func (s *ImageService) PatchWorkflow() *flow.Workflow { return s.patchWorkflow }
// and ImageList concurrently so the size field reflects the same metric shown in the
// image table (docker image ls / docker system df).
func (s *ImageService) GetImageDetail(ctx context.Context, id string) (*imagetypes.DetailSummary, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
var (
inspect image.InspectResponse
listSize int64
@@ -241,8 +237,9 @@ func (s *ImageService) GetImageDetail(ctx context.Context, id string) (*imagetyp
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "image worker")
- var inspectImageErr error
- inspectResult, inspectImageErr := dockerClient.ImageInspect(gctx, id)
+ inspectResult, inspectImageErr := s.dockerService.Call(gctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, id)
+ })
if inspectImageErr != nil {
return fmt.Errorf("inspect not found: %w", inspectImageErr)
}
@@ -294,13 +291,9 @@ func (s *ImageService) GetImageDetail(ctx context.Context, id string) (*imagetyp
}
func (s *ImageService) RemoveImage(ctx context.Context, id string, force bool, user usertypes.Actor) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", id, "", user.ID, user.Username, "0", err, database.JSON{"action": "delete", "force": force})
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- imageDetails, inspectErr := dockerClient.ImageInspect(ctx, id)
+ imageDetails, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, id)
+ })
imageName := func() string {
if inspectErr == nil && len(imageDetails.RepoTags) > 0 {
return imageDetails.RepoTags[0]
@@ -321,21 +314,18 @@ func (s *ImageService) RemoveImage(ctx context.Context, id string, force bool, u
tagReleases = append(tagReleases, s.eventService.BeginDockerResourceSuppressionWindow("image", "", tag))
}
}
- removed, err := dockerClient.ImageRemove(ctx, id, client.ImageRemoveOptions{
- Force: force,
- PruneChildren: true,
+ removed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageRemoveResult, error) {
+ return dockerClient.ImageRemove(ctx, id, client.ImageRemoveOptions{
+ Force: force,
+ PruneChildren: true,
+ })
})
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", id, imageName, user.ID, user.Username, "0", err, database.JSON{"action": "delete", "force": force})
return fmt.Errorf("failed to remove image: %w", err)
}
- s.cleanupImageUpdateRecords(ctx, append(getDeletedImageIDs(removed.Items), id))
-
- // Clean up vulnerability scan records for the deleted image
- if deleteScanResultErr := s.vulnerabilityService.DeleteScanResult(ctx, id); deleteScanResultErr != nil {
- slog.WarnContext(ctx, "failed to delete vulnerability scan record", "id", id, "error", deleteScanResultErr)
- }
+ s.cleanupRemovedImages(ctx, removed.Items, id)
if logErr := s.eventService.LogImageEvent(ctx, event.EventTypeImageDelete, id, imageName, user.ID, user.Username, "0", database.JSON{
"action": "delete",
@@ -348,6 +338,26 @@ func (s *ImageService) RemoveImage(ctx context.Context, id string, force bool, u
return nil
}
+// cleanupRemovedImages drops update and vulnerability records for images Docker deleted or untagged.
+func (s *ImageService) cleanupRemovedImages(ctx context.Context, deleted []image.DeleteResponse, extraIDs ...string) {
+ idsToDelete := make([]string, 0, len(deleted)+len(extraIDs))
+ for _, img := range deleted {
+ if deletedID := cmp.Or(img.Deleted, img.Untagged); deletedID != "" {
+ idsToDelete = append(idsToDelete, deletedID)
+ }
+ }
+ idsToDelete = append(idsToDelete, extraIDs...)
+ if recordsErr := s.imageUpdateService.DeleteRecordsForImages(ctx, idsToDelete); recordsErr != nil {
+ slog.WarnContext(ctx, "failed to clean up image update records", "error", recordsErr)
+ }
+ if orphanErr := s.imageUpdateService.CleanupOrphanedRecords(ctx); orphanErr != nil {
+ slog.WarnContext(ctx, "failed to clean up orphaned image update records", "error", orphanErr)
+ }
+ if deleteErr := s.vulnerabilityService.DeleteScanResultsByImageIDs(ctx, idsToDelete); deleteErr != nil {
+ slog.WarnContext(ctx, "failed to delete vulnerability scan records", "count", len(idsToDelete), "error", deleteErr)
+ }
+}
+
func (s *ImageService) PullImage(ctx context.Context, imageName string, progressWriter io.Writer, user usertypes.Actor, externalCreds []containerregistry.Credential) error {
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
@@ -428,24 +438,69 @@ func (s *ImageService) PullImage(ctx context.Context, imageName string, progress
// engine doesn't report it. Pull-policy refresh windows compare against this
// timestamp, the same clock compose v5.5.0 uses.
func (s *ImageService) ImageLastTagTime(ctx context.Context, imageName string) (time.Time, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return time.Time{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- inspect, err := dockerClient.ImageInspect(ctx, imageName)
+ inspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageName)
+ })
if err != nil {
return time.Time{}, fmt.Errorf("failed to inspect image %s: %w", imageName, err)
}
return inspect.Metadata.LastTagTime, nil
}
-func (s *ImageService) ReconcilePulledImageUpdate(ctx context.Context, imageName string) error {
- if s.imageUpdateService == nil {
- return nil
+// PullRequest describes one pull through Pull.
+type PullRequest struct {
+ // Ref is the reference pulled from the registry.
+ Ref string
+ // Tag, when set, is the reference Ref is tagged as after the pull; it is the one reconciled.
+ Tag string
+ Progress io.Writer
+ User usertypes.Actor
+ Credentials []containerregistry.Credential
+}
+
+// Pull pulls req.Ref within the dockerImagePullTimeout setting and reconciles the pulled reference's update state.
+func (s *ImageService) Pull(ctx context.Context, req PullRequest) error {
+ pullCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))
+ defer cancel()
+
+ pull := func(ctx context.Context) error {
+ return s.PullImage(ctx, req.Ref, req.Progress, req.User, req.Credentials)
}
+ pullErr := func() error {
+ if req.Tag == "" || req.Tag == req.Ref {
+ return pull(pullCtx)
+ }
+ // A digest pull is tagged back to the reference the caller selected.
+ dockerClient, clientErr := s.dockerService.GetClient(pullCtx)
+ if clientErr != nil {
+ return clientErr
+ }
+ return imageref.PullAsTag(pullCtx, dockerClient, req.Ref, req.Tag, pull)
+ }()
+ if pullErr != nil {
+ if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {
+ return fmt.Errorf("image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting): %w", req.Ref, context.DeadlineExceeded)
+ }
+ return fmt.Errorf("failed to pull image %s: %w", req.Ref, pullErr)
+ }
+
+ // Reconcile on the caller's context so a late pull still clears its update records (#4306).
+ if reconcileErr := s.imageUpdateService.MarkImageRefUpToDateAfterPull(ctx, cmp.Or(req.Tag, req.Ref)); reconcileErr != nil {
+ slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", cmp.Or(req.Tag, req.Ref), "error", reconcileErr)
+ }
+ return nil
+}
+
+// RemoteDigest resolves imageRef's registry digest within the registry timeout setting.
+func (s *ImageService) RemoteDigest(ctx context.Context, imageRef string, credentials []containerregistry.Credential) (string, error) {
+ digestCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().RegistryTimeout.AsInt(), timeouts.DefaultRegistry))
+ defer cancel()
- return s.imageUpdateService.MarkImageRefUpToDateAfterPull(ctx, imageName)
+ result, err := s.registryService.InspectImageDigest(digestCtx, imageRef, credentials)
+ if err != nil {
+ return "", err
+ }
+ return kit.FromPtr(result).Digest, nil
}
// TagImage adds a repository tag to an existing image.
@@ -462,14 +517,10 @@ func (s *ImageService) TagImage(ctx context.Context, source string, req imagetyp
target := kit.Ternary(tag != "", repository+":"+tag, repository)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", "", source, user.ID, user.Username, "0", err, database.JSON{"action": "tag", "target": target})
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
defer s.eventService.BeginDockerResourceSuppressionWindow("image", "", target)()
- _, err = dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: source, Target: target})
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageTagResult, error) {
+ return dockerClient.ImageTag(ctx, client.ImageTagOptions{Source: source, Target: target})
+ })
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeImageError, "image", "", source, user.ID, user.Username, "0", err, database.JSON{"action": "tag", "target": target})
return fmt.Errorf("failed to tag image: %w", err)
@@ -495,12 +546,9 @@ func (s *ImageService) GetImageHistory(ctx context.Context, imageName string) ([
return nil, errors.New("image name is required")
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- result, err := dockerClient.ImageHistory(ctx, imageName)
+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageHistoryResult, error) {
+ return dockerClient.ImageHistory(ctx, imageName)
+ })
if err != nil {
return nil, fmt.Errorf("failed to get image history: %w", err)
}
@@ -526,12 +574,9 @@ func (s *ImageService) SearchImages(ctx context.Context, term string) ([]imagety
return nil, errors.New("search term is required")
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- result, err := dockerClient.ImageSearch(ctx, term, client.ImageSearchOptions{})
+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageSearchResult, error) {
+ return dockerClient.ImageSearch(ctx, term, client.ImageSearchOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to search images: %w", err)
}
@@ -622,12 +667,9 @@ func (s *ImageService) LoadImageFromReader(ctx context.Context, reader io.Reader
}
func (s *ImageService) ImageExistsLocally(ctx context.Context, imageName string) (bool, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return false, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- _, err = dockerClient.ImageInspect(ctx, imageName)
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageName)
+ })
if err == nil {
return true, nil
}
@@ -684,11 +726,6 @@ func ShouldRetryAnonymousPull(pullOptions client.ImagePullOptions, pullErr error
}
func (s *ImageService) PruneImages(ctx context.Context, options system.PruneImagesOptions) (*image.PruneReport, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
filterArgs, err := func() (client.Filters, error) {
switch options.Mode {
case system.PruneImageModeNone:
@@ -710,18 +747,16 @@ func (s *ImageService) PruneImages(ctx context.Context, options system.PruneImag
return nil, err
}
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to prune images: %w", err)
+ }
report, err := dockerClient.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})
if err != nil {
return nil, fmt.Errorf("failed to prune images: %w", err)
}
- idsToDelete := getDeletedImageIDs(report.Report.ImagesDeleted)
- s.cleanupImageUpdateRecords(ctx, idsToDelete)
- if len(idsToDelete) > 0 {
- if deleteErr := s.vulnerabilityService.DeleteScanResultsByImageIDs(ctx, idsToDelete); deleteErr != nil {
- slog.WarnContext(ctx, "failed to delete vulnerability scan records after prune", "count", len(idsToDelete), "error", deleteErr)
- }
- }
+ s.cleanupRemovedImages(ctx, report.Report.ImagesDeleted)
if logErr := s.eventService.LogImageEvent(ctx, event.EventTypeImageDelete, "", "bulk_prune", usertypes.SystemUser.ID, usertypes.SystemUser.Username, "0", database.JSON{
"action": "prune",
@@ -736,33 +771,6 @@ func (s *ImageService) PruneImages(ctx context.Context, options system.PruneImag
return &report.Report, nil
}
-func getDeletedImageIDs(deletedImages []image.DeleteResponse) []string {
- if len(deletedImages) == 0 {
- return nil
- }
-
- idsToDelete := make([]string, 0, len(deletedImages))
- for _, img := range deletedImages {
- if img.Deleted != "" {
- idsToDelete = append(idsToDelete, img.Deleted)
- continue
- }
- if img.Untagged != "" {
- idsToDelete = append(idsToDelete, img.Untagged)
- }
- }
- return idsToDelete
-}
-
-func (s *ImageService) cleanupImageUpdateRecords(ctx context.Context, idsToDelete []string) {
- if err := s.imageUpdateService.DeleteRecordsForImages(ctx, idsToDelete); err != nil {
- slog.WarnContext(ctx, "failed to clean up image update records", "error", err)
- }
- if err := s.imageUpdateService.CleanupOrphanedRecords(ctx); err != nil {
- slog.WarnContext(ctx, "failed to clean up orphaned image update records", "error", err)
- }
-}
-
// GetUpdateInfoByImageIDs returns a map of image ID to UpdateInfo for the given image IDs.
// This is used by the container service to populate update info for containers.
func (s *ImageService) GetUpdateInfoByImageIDs(ctx context.Context, imageIDs []string) (map[string]*imagetypes.UpdateInfo, error) {
@@ -813,16 +821,7 @@ func (s *ImageService) GetUpdateInfoByContainers(ctx context.Context, containers
}
if len(digestImageIDs) > 0 {
- imageIDs := make([]string, 0, len(digestImageIDs))
- seen := make(map[string]struct{}, len(digestImageIDs))
- for _, imageID := range digestImageIDs {
- if _, exists := seen[imageID]; exists {
- continue
- }
- seen[imageID] = struct{}{}
- imageIDs = append(imageIDs, imageID)
- }
- byImageID, err := s.GetUpdateInfoByImageIDs(ctx, imageIDs)
+ byImageID, err := s.GetUpdateInfoByImageIDs(ctx, kit.Unique(slices.Collect(maps.Values(digestImageIDs))))
if err != nil {
return nil, err
}
@@ -879,23 +878,12 @@ func (s *ImageService) GetUpdateInfoByImageRefs(ctx context.Context, imageRefs [
}
repositoryCandidates := make([]string, 0)
- repositorySeen := make(map[string]struct{})
tags := make([]string, 0, len(lookups))
- tagSeen := make(map[string]struct{})
-
for _, lookup := range lookups {
- if _, exists := tagSeen[lookup.tag]; !exists {
- tagSeen[lookup.tag] = struct{}{}
- tags = append(tags, lookup.tag)
- }
- for repo := range lookup.repositoryCandidates {
- if _, exists := repositorySeen[repo]; exists {
- continue
- }
- repositorySeen[repo] = struct{}{}
- repositoryCandidates = append(repositoryCandidates, repo)
- }
+ tags = append(tags, lookup.tag)
+ repositoryCandidates = slices.AppendSeq(repositoryCandidates, maps.Keys(lookup.repositoryCandidates))
}
+ repositoryCandidates, tags = kit.Unique(repositoryCandidates), kit.Unique(tags)
if len(repositoryCandidates) == 0 || len(tags) == 0 {
return result, nil
@@ -913,7 +901,10 @@ func (s *ImageService) GetUpdateInfoByImageRefs(ctx context.Context, imageRefs [
// earliest position, matching the order the records were fetched in.
index := make(map[imageUpdateRecordKey]int, len(updateRecords))
for i := range updateRecords {
- key := newImageUpdateRecordKey(updateRecords[i].Repository, updateRecords[i].Tag)
+ key := imageUpdateRecordKey{
+ repository: strings.TrimSpace(updateRecords[i].Repository),
+ tag: strings.ToLower(strings.TrimSpace(updateRecords[i].Tag)),
+ }
if existing, ok := index[key]; !ok || updateRecords[i].CheckTime.After(updateRecords[existing].CheckTime) {
index[key] = i
}
@@ -922,7 +913,7 @@ func (s *ImageService) GetUpdateInfoByImageRefs(ctx context.Context, imageRefs [
// Pick the latest record across the lookup's repository aliases; equal check times fall back to fetch order.
latest := -1
for repository := range lookup.repositoryCandidates {
- i, ok := index[newImageUpdateRecordKey(repository, lookup.tag)]
+ i, ok := index[imageUpdateRecordKey{repository: strings.TrimSpace(repository), tag: strings.ToLower(strings.TrimSpace(lookup.tag))}]
if !ok {
continue
}
@@ -1066,10 +1057,6 @@ func (s *ImageService) AggregateImageCounts(ctx context.Context) (base.ApiRespon
}, nil
}
-func newImageUpdateRecordKey(repository, tag string) imageUpdateRecordKey {
- return imageUpdateRecordKey{repository: strings.TrimSpace(repository), tag: strings.ToLower(strings.TrimSpace(tag))}
-}
-
// BuildProjectIDMap returns a map of compose project name → project ID for any
// containers that carry the com.docker.compose.project label. The lookup uses a
// short-TTL cache shared across all callers of this ImageService instance.
@@ -1248,10 +1235,8 @@ func MapDockerImagesToDTOs(
}
return di.RepoTags[0], "latest"
}
- for _, rd := range di.RepoDigests {
- if candidateRepo, _, found := strings.CutLast(rd, "@"); rd != "<none>@<none>" && found && candidateRepo != "" {
- return candidateRepo, "<none>"
- }
+ if candidateRepo, found := imageref.RepoDigestRepository(di.RepoDigests); found {
+ return candidateRepo, "<none>"
}
return "<none>", "<none>"
}()
diff --git a/backend/internal/imageupdate/service.go b/backend/internal/imageupdate/service.go
index aadfa84c2..07aadb249 100644
--- a/backend/internal/imageupdate/service.go
+++ b/backend/internal/imageupdate/service.go
@@ -171,13 +171,6 @@ func NewImageUpdateService(
}
}
-func (s *ImageUpdateService) dockerAPIContext(ctx context.Context) (context.Context, context.CancelFunc) {
- if s.settingsService == nil {
- return context.WithTimeout(ctx, timeouts.DefaultDockerAPI)
- }
- return context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
-}
-
func (s *ImageUpdateService) registryContext(ctx context.Context) (context.Context, context.CancelFunc) {
if s.settingsService == nil {
return context.WithTimeout(ctx, timeouts.DefaultRegistry)
@@ -185,19 +178,7 @@ func (s *ImageUpdateService) registryContext(ctx context.Context) (context.Conte
return context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().RegistryTimeout.AsInt(), timeouts.DefaultRegistry))
}
-func (s *ImageUpdateService) dockerClient(ctx context.Context) (*client.Client, error) {
- if s.dockerService == nil {
- return nil, errors.New("docker service unavailable")
- }
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
- return dockerClient, nil
-}
-
func (s *ImageUpdateService) composeBuildImageRefs(ctx context.Context) (map[string]struct{}, error) {
- buildRefs := make(map[string]struct{})
var projectRows []struct {
ID string
BuildImageRefsJSON *string `gorm:"column:build_image_refs_json"`
@@ -210,15 +191,6 @@ func (s *ImageUpdateService) composeBuildImageRefs(ctx context.Context) (map[str
return nil, fmt.Errorf("load project build image references: %w", err)
}
- for _, row := range projectRows {
- for _, imageRef := range projects.ParseImageRefsJSON(kit.FromPtr(row.BuildImageRefsJSON)) {
- normalized := refs.NormalizeImageUpdateRef(imageRef)
- if normalized != "" {
- buildRefs[normalized] = struct{}{}
- }
- }
- }
-
// Copacetic-patched tags only exist in the local daemon, so treat them like
// locally built images instead of failing a remote registry lookup. Never
// fail the update check over patch history.
@@ -230,9 +202,13 @@ func (s *ImageUpdateService) composeBuildImageRefs(ctx context.Context) (map[str
Pluck("patched_ref", &patchedRefs).Error; err != nil {
slog.DebugContext(ctx, "failed to load patched image references", "error", err)
}
+ for _, row := range projectRows {
+ patchedRefs = append(patchedRefs, projects.ParseImageRefsJSON(kit.FromPtr(row.BuildImageRefsJSON))...)
+ }
+
+ buildRefs := make(map[string]struct{}, len(patchedRefs))
for _, imageRef := range patchedRefs {
- normalized := refs.NormalizeImageUpdateRef(imageRef)
- if normalized != "" {
+ if normalized := refs.NormalizeImageUpdateRef(imageRef); normalized != "" {
buildRefs[normalized] = struct{}{}
}
}
@@ -404,7 +380,13 @@ func (s *ImageUpdateService) CheckImageUpdate(ctx context.Context, imageRef stri
if kit.FromPtr(snapshot).ImageID != "" {
return snapshot.ImageID, nil
}
- return s.getImageIDByRef(ctx, imageRef)
+ inspectResponse, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageRef)
+ })
+ if inspectErr != nil {
+ return "", fmt.Errorf("image not found: %w", inspectErr)
+ }
+ return inspectResponse.ID, nil
}()
switch {
case imageIDErr != nil:
@@ -533,18 +515,12 @@ func (s *ImageUpdateService) parseImageReference(imageRef string) *ImageParts {
}
func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string) (string, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return "", err
- }
-
imageID = strings.TrimPrefix(imageID, "sha256:")
if imageRef, ok := func() (string, bool) {
- apiCtx, cancel := s.dockerAPIContext(ctx)
- defer cancel()
-
- inspectResponse, inspectErr := dockerClient.ImageInspect(apiCtx, imageID)
+ inspectResponse, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageID)
+ })
if inspectErr != nil {
return "", false
}
@@ -556,12 +532,8 @@ func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string
}
}
}
- for _, digestValue := range inspectResponse.RepoDigests {
- if digestValue != "<none>@<none>" {
- if repo, _, found := strings.Cut(digestValue, "@"); found {
- return repo + ":latest", true
- }
- }
+ if repo, found := imageref.RepoDigestRepository(inspectResponse.RepoDigests); found {
+ return repo + ":latest", true
}
return "", false
}(); ok {
@@ -571,10 +543,9 @@ func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string
// Fallback: if the image was pruned, look up the image reference from
// running containers that were started from this image ID.
if imageRef, ok := func() (string, bool) {
- apiCtx, cancel := s.dockerAPIContext(ctx)
- defer cancel()
-
- containers, listErr := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
+ containers, listErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if listErr != nil {
return "", false
}
@@ -595,21 +566,15 @@ func (s *ImageUpdateService) getImageRefByID(ctx context.Context, imageID string
}
func (s *ImageUpdateService) getAllImageRefs(ctx context.Context, limit int) ([]string, error) {
- imageCtx, cancelImage := s.dockerAPIContext(ctx)
- images, err := s.dockerService.ListImages(imageCtx)
- cancelImage()
+ images, err := s.dockerService.ListImages(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list Docker images: %w", err)
}
- containerCtx, cancelContainers := s.dockerAPIContext(ctx)
- containers, err := s.dockerService.ListContainers(containerCtx)
- cancelContainers()
+ containers, err := s.dockerService.ListContainers(ctx)
if err != nil {
slog.WarnContext(ctx, "failed to list Docker containers; continuing without update-check opt-out filtering", "error", err.Error())
- return filterImageSummariesByContainerOptOut(images, nil, limit), nil
}
-
return filterImageSummariesByContainerOptOut(images, containers, limit), nil
}
@@ -695,15 +660,9 @@ func filterImageSummariesByContainerOptOut(images []image.Summary, containers []
}
func (s *ImageUpdateService) inspectLocalImageSnapshot(ctx context.Context, imageRef string, composeBuildRefs map[string]struct{}) (*localImageSnapshot, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, err
- }
-
- apiCtx, cancel := s.dockerAPIContext(ctx)
- defer cancel()
-
- inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageRef)
+ inspectResponse, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageRef)
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect image: %w", err)
}
@@ -725,12 +684,12 @@ func (s *ImageUpdateService) inspectLocalImageSnapshot(ctx context.Context, imag
}
}
- repo, tag := extractRepoAndTagFromImage(inspectResponse.InspectResponse)
+ repo, tag := extractRepoAndTagFromImage(inspectResponse.InspectResponse, s.parseImageReference(imageRef))
return &localImageSnapshot{
ImageID: inspectResponse.ID,
Repository: repo,
- Tag: tagWithFallback(tag, s.parseImageReference(imageRef)),
+ Tag: tag,
// Use the first repository digest, else the first digest; fallback to image ID if no repo digests available
PrimaryDigest: slices.Concat(repositoryDigests, allDigests, []string{inspectResponse.ID})[0],
AllDigests: kit.Ternary(len(allDigests) > 0, allDigests, []string{inspectResponse.ID}),
@@ -756,7 +715,17 @@ func (s *ImageUpdateService) CheckImageUpdateByID(ctx context.Context, imageID s
return nil, err
}
if len(result.ContainerUpdates) == 0 {
- if saveErr := s.saveUpdateResultByID(ctx, imageID, result, s.parseImageReference(imageRef)); saveErr != nil {
+ saveErr := func() error {
+ dockerImage, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageID)
+ })
+ if inspectErr != nil {
+ return fmt.Errorf("failed to inspect image: %w", inspectErr)
+ }
+ repo, tag := extractRepoAndTagFromImage(dockerImage.InspectResponse, s.parseImageReference(imageRef))
+ return s.savePreparedUpdateResult(ctx, imageID, repo, tag, result)
+ }()
+ if saveErr != nil {
slog.WarnContext(ctx, "Failed to save update result by ID", "imageId", imageID, "error", saveErr.Error())
}
}
@@ -772,7 +741,9 @@ func (s *ImageUpdateService) saveUpdateResultWithSnapshot(ctx context.Context, i
if parts == nil {
return errors.New("invalid image reference")
}
- imageID, err := s.getImageIDByRef(ctx, imageRef)
+ inspectResponse, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, imageRef)
+ })
if err != nil {
repository := func() string {
repositoryPath := strings.TrimSpace(parts.Repository)
@@ -796,10 +767,13 @@ func (s *ImageUpdateService) saveUpdateResultWithSnapshot(ctx context.Context, i
return s.savePreparedUpdateResult(ctx, syntheticID, repository, parts.Tag, result)
}
- return s.saveUpdateResultByID(ctx, imageID, result, parts)
+ repo, tag := extractRepoAndTagFromImage(inspectResponse.InspectResponse, parts)
+ return s.savePreparedUpdateResult(ctx, inspectResponse.ID, repo, tag, result)
}
-func extractRepoAndTagFromImage(dockerImage image.InspectResponse) (repo, tag string) {
+// extractRepoAndTagFromImage reads the image's first usable tag; an untagged image uses fallback's tag when set.
+func extractRepoAndTagFromImage(dockerImage image.InspectResponse, fallback *ImageParts) (repo, tag string) {
+ untagged := cmp.Or(strings.TrimSpace(kit.FromPtr(fallback).Tag), "<none>")
if len(dockerImage.RepoTags) > 0 && dockerImage.RepoTags[0] != "<none>:<none>" {
if named, err := reference.ParseNormalizedNamed(dockerImage.RepoTags[0]); err == nil {
if tagged, ok := named.(reference.NamedTagged); ok {
@@ -815,16 +789,11 @@ func extractRepoAndTagFromImage(dockerImage image.InspectResponse) (repo, tag st
return parts[0], "latest"
}
- for _, rd := range dockerImage.RepoDigests {
- if rd == "<none>@<none>" {
- continue
- }
- if repoCandidate, _, found := strings.CutLast(rd, "@"); found && repoCandidate != "" {
- return repoCandidate, "<none>"
- }
+ if repoCandidate, found := imageref.RepoDigestRepository(dockerImage.RepoDigests); found {
+ return repoCandidate, untagged
}
- return "<none>", "<none>"
+ return "<none>", untagged
}
func buildImageUpdateRecord(imageID, repo, tag string, result *imageupdate.Response, notificationSent bool, latestSeenAt *time.Time, carried *ImageUpdateRecord) *ImageUpdateRecord {
@@ -904,53 +873,12 @@ func savePreparedUpdateResultWithTx(tx *gorm.DB, imageID, repo, tag string, resu
)).Error
}
-func (s *ImageUpdateService) saveUpdateResultByID(ctx context.Context, imageID string, result *imageupdate.Response, fallback *ImageParts) error {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return err
- }
-
- apiCtx, cancel := s.dockerAPIContext(ctx)
- defer cancel()
-
- dockerImage, err := dockerClient.ImageInspect(apiCtx, imageID)
- if err != nil {
- return fmt.Errorf("failed to inspect image: %w", err)
- }
-
- repo, tag := extractRepoAndTagFromImage(dockerImage.InspectResponse)
- return s.savePreparedUpdateResult(ctx, imageID, repo, tagWithFallback(tag, fallback), result)
-}
-
-func tagWithFallback(tag string, fallback *ImageParts) string {
- if tag == "<none>" && strings.TrimSpace(kit.FromPtr(fallback).Tag) != "" {
- return fallback.Tag
- }
- return tag
-}
-
func (s *ImageUpdateService) savePreparedUpdateResult(ctx context.Context, imageID, repo, tag string, result *imageupdate.Response) error {
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
return savePreparedUpdateResultWithTx(tx, imageID, repo, tag, result)
})
}
-func (s *ImageUpdateService) getImageIDByRef(ctx context.Context, imageRef string) (string, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return "", err
- }
-
- apiCtx, cancel := s.dockerAPIContext(ctx)
- defer cancel()
-
- inspectResponse, err := dockerClient.ImageInspect(apiCtx, imageRef)
- if err != nil {
- return "", fmt.Errorf("image not found: %w", err)
- }
- return inspectResponse.ID, nil
-}
-
func (s *ImageUpdateService) MarkImageRefUpToDateAfterPull(ctx context.Context, imageRef string) error {
snapshot, err := s.inspectLocalImageSnapshot(ctx, imageRef, nil)
if err != nil {
@@ -1404,13 +1332,9 @@ func (s *ImageUpdateService) CheckImages(ctx context.Context, request imageupdat
// MonitoredImageRefs keeps the refs whose containers still permit update checks.
func (s *ImageUpdateService) MonitoredImageRefs(ctx context.Context, imageRefs []string) ([]string, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, err
- }
- apiCtx, cancel := s.dockerAPIContext(ctx)
- listed, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
- cancel()
+ listed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if err != nil {
return nil, fmt.Errorf("list containers for update-check eligibility: %w", err)
}
@@ -1683,13 +1607,9 @@ func (s *ImageUpdateService) SendBatchUpdateNotifications(ctx context.Context) e
// Records for resources that opted out of monitoring since the check, or
// whose container is gone, stay unnotified: they resurface if monitoring
// is re-enabled and are otherwise removed by orphan cleanup.
- dockerClient, clientErr := s.dockerClient(notifCtx)
- if clientErr != nil {
- return records, fmt.Errorf("resolve update-check eligibility: %w", clientErr)
- }
- apiCtx, apiCancel := s.dockerAPIContext(notifCtx)
- listed, listErr := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
- apiCancel()
+ listed, listErr := s.dockerService.Call(notifCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if listErr != nil {
return records, fmt.Errorf("list containers for update-check eligibility: %w", listErr)
}
@@ -1869,13 +1789,9 @@ func (s *ImageUpdateService) checkContainerTagUpdates(ctx context.Context, image
if len(wanted) == 0 {
return map[string]*imageupdate.Response{}, nil
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return map[string]*imageupdate.Response{}, err
- }
- apiCtx, cancel := s.dockerAPIContext(ctx)
- listed, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})
- cancel()
+ listed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if err != nil {
return map[string]*imageupdate.Response{}, fmt.Errorf("list containers for tag update checks: %w", err)
}
diff --git a/backend/internal/imageupdate/service_test.go b/backend/internal/imageupdate/service_test.go
index 8619c3bca..7894e8f60 100644
--- a/backend/internal/imageupdate/service_test.go
+++ b/backend/internal/imageupdate/service_test.go
@@ -781,7 +781,7 @@ func TestImageUpdateService_CheckImageUpdate_SkipsDigestPinnedReference(t *testi
pinnedDigest := digest.FromString("pinned-newt").String()
imageRef := "ghcr.io/fosrl/newt@" + pinnedDigest
eventService := event.NewEventService(db, nil, nil)
- svc := NewImageUpdateService(db, nil, nil, nil, eventService, notification.NewNotificationService(db, nil, nil, nil, nil), nil)
+ svc := NewImageUpdateService(db, nil, nil, newImageUpdateNotificationDockerService(t, nil), eventService, notification.NewNotificationService(db, nil, nil, nil, nil), nil)
result, err := svc.CheckImageUpdate(t.Context(), imageRef)
require.NoError(t, err)
@@ -797,7 +797,7 @@ func TestImageUpdateService_CheckImages_SkipsDigestPinnedReference(t *testing.T)
db := setupImageUpdateTestDB(t)
pinnedDigest := digest.FromString("batch-pinned-newt").String()
imageRef := "ghcr.io/fosrl/newt@" + pinnedDigest
- svc := NewImageUpdateService(db, nil, nil, nil, nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)
+ svc := NewImageUpdateService(db, nil, nil, newImageUpdateNotificationDockerService(t, nil), nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)
require.Equal(t, [][]string{{imageRef}}, svc.groupImageRefs([]string{imageRef}))
@@ -828,7 +828,7 @@ func TestImageUpdateService_CheckImages_SkippedDigestPinnedReferenceClearsStaleE
LastError: new("failed to get remote digest"),
CheckTime: time.Now().Add(-time.Hour),
}).Error)
- svc := NewImageUpdateService(db, nil, nil, nil, nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)
+ svc := NewImageUpdateService(db, nil, nil, newImageUpdateNotificationDockerService(t, nil), nil, notification.NewNotificationService(db, nil, nil, nil, nil), nil)
results, err := checkImages(t, t.Context(), svc, imageRef)
require.NoError(t, err)
@@ -1126,7 +1126,7 @@ func TestImageUpdateService_GetAllImageRefsUsesDockerAPITimeout(t *testing.T) {
db,
settingsService,
nil,
- &docker.DockerClientService{Client: newImageUpdateTestDockerClient(t, server)},
+ docker.NewDockerClientService(t.Context(), nil, nil, settingsService).WithClient(newImageUpdateTestDockerClient(t, server)),
nil,
notification.NewNotificationService(db, nil, nil, nil, nil),
nil,
@@ -1152,7 +1152,7 @@ func TestImageUpdateService_InspectLocalImageSnapshotUsesDockerAPITimeout(t *tes
db,
settingsService,
nil,
- &docker.DockerClientService{Client: newImageUpdateTestDockerClient(t, server)},
+ docker.NewDockerClientService(t.Context(), nil, nil, settingsService).WithClient(newImageUpdateTestDockerClient(t, server)),
nil,
notification.NewNotificationService(db, nil, nil, nil, nil),
nil,
@@ -1962,10 +1962,14 @@ func TestImageUpdateService_SendBatchNotifications_UnresolvedEligibilityLeavesUn
calls := newImageUpdateGenericWebhookProvider(t, db)
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
- svc := NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ t.Cleanup(server.Close)
+ svc := NewImageUpdateService(db, nil, nil, &docker.DockerClientService{Client: newImageUpdateTestDockerClient(t, server)}, nil, notif, nil)
require.NoError(t, db.Create(&ImageUpdateRecord{ID: "sha256:img", Repository: "test/repo", Tag: "latest", HasUpdate: true}).Error)
- require.ErrorContains(t, svc.SendBatchUpdateNotifications(t.Context()), "resolve update-check eligibility: docker service unavailable")
+ require.ErrorContains(t, svc.SendBatchUpdateNotifications(t.Context()), "list containers for update-check eligibility")
require.Zero(t, calls.Load(), "nothing is sent when eligibility cannot be resolved")
var reloaded ImageUpdateRecord
diff --git a/backend/internal/network/service.go b/backend/internal/network/service.go
index 2708831e7..b3c581d8e 100644
--- a/backend/internal/network/service.go
+++ b/backend/internal/network/service.go
@@ -69,12 +69,9 @@ func NewNetworkService(db *database.DB, dockerService *dockerInternal.DockerClie
}
func (s *NetworkService) GetNetworkByID(ctx context.Context, id string) (*network.Inspect, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- networkInspect, err := compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})
+ networkInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {
+ return compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("network not found: %w", err)
}
@@ -122,11 +119,6 @@ func (s *NetworkService) GetNetworkTopology(ctx context.Context) (*networktypes.
}
func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topology, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
containers, err := s.dockerService.ListContainers(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list containers: %w", err)
@@ -137,7 +129,9 @@ func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topol
containersByID[rawContainer.ID] = rawContainer
}
- networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ networkList, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkListResult, error) {
+ return compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list Docker networks: %w", err)
}
@@ -151,7 +145,9 @@ func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topol
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "network worker")
- inspected, inspectErr := compat.NetworkInspectWithCompatibility(groupCtx, dockerClient, rawNetwork.ID, client.NetworkInspectOptions{})
+ inspected, inspectErr := s.dockerService.Call(groupCtx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {
+ return compat.NetworkInspectWithCompatibility(ctx, dockerClient, rawNetwork.ID, client.NetworkInspectOptions{})
+ })
if inspectErr != nil {
return fmt.Errorf("failed to inspect network %s: %w", rawNetwork.Name, inspectErr)
}
@@ -219,14 +215,10 @@ func (s *NetworkService) buildTopology(ctx context.Context) (*networktypes.Topol
}
func (s *NetworkService) CreateNetwork(ctx context.Context, name string, options client.NetworkCreateOptions, user usertypes.Actor) (*network.CreateResponse, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", "", name, user.ID, user.Username, "0", err, database.JSON{"action": "create", "driver": options.Driver})
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
defer s.eventService.BeginDockerResourceSuppressionWindow("network", "", name)()
- response, err := dockerClient.NetworkCreate(ctx, name, options)
+ response, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkCreateResult, error) {
+ return dockerClient.NetworkCreate(ctx, name, options)
+ })
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", "", name, user.ID, user.Username, "0", err, database.JSON{"action": "create", "driver": options.Driver})
return nil, fmt.Errorf("failed to create network: %w", err)
@@ -248,17 +240,15 @@ func (s *NetworkService) CreateNetwork(ctx context.Context, name string, options
}
func (s *NetworkService) RemoveNetwork(ctx context.Context, id string, user usertypes.Actor) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", id, "", user.ID, user.Username, "0", err, database.JSON{"action": "delete"})
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- networkInfo, err := compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})
+ networkInfo, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {
+ return compat.NetworkInspectWithCompatibility(ctx, dockerClient, id, client.NetworkInspectOptions{})
+ })
networkName := kit.Ternary(err == nil, networkInfo.Network.Name, id)
defer s.eventService.BeginDockerResourceSuppressionWindow("network", id, networkName)()
- if _, networkRemoveErr := dockerClient.NetworkRemove(ctx, id, client.NetworkRemoveOptions{}); networkRemoveErr != nil {
+ if _, networkRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkRemoveResult, error) {
+ return dockerClient.NetworkRemove(ctx, id, client.NetworkRemoveOptions{})
+ }); networkRemoveErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", id, networkName, user.ID, user.Username, "0", networkRemoveErr, database.JSON{"action": "delete"})
return fmt.Errorf("failed to remove network: %w", networkRemoveErr)
}
@@ -276,12 +266,6 @@ func (s *NetworkService) RemoveNetwork(ctx context.Context, id string, user user
// ConnectContainer connects a container to a network without recreating it.
func (s *NetworkService) ConnectContainer(ctx context.Context, networkID string, req networktypes.ConnectContainerRequest, user usertypes.Actor) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", networkID, "", user.ID, user.Username, "0", err, database.JSON{"action": "connect", "containerId": req.ContainerID})
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
ipv4, ipv4Err := netip.ParseAddr(strings.TrimSpace(req.IPv4Address))
if ipv4Err != nil && strings.TrimSpace(req.IPv4Address) != "" {
return common.Classify(common.ErrValidation, fmt.Errorf("invalid IPv4 address %s: %w", strings.TrimSpace(req.IPv4Address), ipv4Err))
@@ -291,20 +275,22 @@ func (s *NetworkService) ConnectContainer(ctx context.Context, networkID string,
return common.Classify(common.ErrValidation, fmt.Errorf("invalid IPv6 address %s: %w", strings.TrimSpace(req.IPv6Address), ipv6Err))
}
- if _, networkConnectErr := dockerClient.NetworkConnect(ctx, networkID, client.NetworkConnectOptions{
- Container: req.ContainerID,
- EndpointConfig: &network.EndpointSettings{
- Aliases: req.Aliases,
- IPAMConfig: func() *network.EndpointIPAMConfig {
- if !ipv4.IsValid() && !ipv6.IsValid() {
- return nil
- }
- return &network.EndpointIPAMConfig{
- IPv4Address: ipv4,
- IPv6Address: ipv6,
- }
- }(),
- },
+ if _, networkConnectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkConnectResult, error) {
+ return dockerClient.NetworkConnect(ctx, networkID, client.NetworkConnectOptions{
+ Container: req.ContainerID,
+ EndpointConfig: &network.EndpointSettings{
+ Aliases: req.Aliases,
+ IPAMConfig: func() *network.EndpointIPAMConfig {
+ if !ipv4.IsValid() && !ipv6.IsValid() {
+ return nil
+ }
+ return &network.EndpointIPAMConfig{
+ IPv4Address: ipv4,
+ IPv6Address: ipv6,
+ }
+ }(),
+ },
+ })
}); networkConnectErr != nil {
s.eventService.LogErrorEvent(
ctx,
@@ -338,15 +324,11 @@ func (s *NetworkService) ConnectContainer(ctx context.Context, networkID string,
// DisconnectContainer disconnects a container from a network.
func (s *NetworkService) DisconnectContainer(ctx context.Context, networkID string, req networktypes.DisconnectContainerRequest, user usertypes.Actor) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- s.eventService.LogErrorEvent(ctx, event.EventTypeNetworkError, "network", networkID, "", user.ID, user.Username, "0", err, database.JSON{"action": "disconnect", "containerId": req.ContainerID})
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, networkDisconnectErr := dockerClient.NetworkDisconnect(ctx, networkID, client.NetworkDisconnectOptions{
- Container: req.ContainerID,
- Force: req.Force,
+ if _, networkDisconnectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkDisconnectResult, error) {
+ return dockerClient.NetworkDisconnect(ctx, networkID, client.NetworkDisconnectOptions{
+ Container: req.ContainerID,
+ Force: req.Force,
+ })
}); networkDisconnectErr != nil {
s.eventService.LogErrorEvent(
ctx,
@@ -380,12 +362,9 @@ func (s *NetworkService) DisconnectContainer(ctx context.Context, networkID stri
}
func (s *NetworkService) PruneNetworks(ctx context.Context) (*network.PruneReport, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- report, err := dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: make(client.Filters)})
+ report, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkPruneResult, error) {
+ return dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: make(client.Filters)})
+ })
if err != nil {
return nil, fmt.Errorf("failed to prune networks: %w", err)
}
@@ -402,11 +381,6 @@ func (s *NetworkService) PruneNetworks(ctx context.Context) (*network.PruneRepor
}
func (s *NetworkService) ListNetworksPaginated(ctx context.Context, params pagination.QueryParams) ([]networktypes.Summary, pagination.Response, networktypes.UsageCounts, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, networktypes.UsageCounts{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
containers, err := s.dockerService.ListContainers(ctx)
if err != nil {
return nil, pagination.Response{}, networktypes.UsageCounts{}, fmt.Errorf("failed to list containers: %w", err)
@@ -414,7 +388,9 @@ func (s *NetworkService) ListNetworksPaginated(ctx context.Context, params pagin
inUseByID, inUseByName := docker.BuildNetworkUsageMaps(containers)
- networkList, err := compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ networkList, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkListResult, error) {
+ return compat.NetworkListWithCompatibility(ctx, dockerClient, client.NetworkListOptions{})
+ })
if err != nil {
return nil, pagination.Response{}, networktypes.UsageCounts{}, fmt.Errorf("failed to list Docker networks: %w", err)
}
diff --git a/backend/internal/project/children/deployment/service.go b/backend/internal/project/children/deployment/service.go
index 8b877ad83..fd1848868 100644
--- a/backend/internal/project/children/deployment/service.go
+++ b/backend/internal/project/children/deployment/service.go
@@ -24,8 +24,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
@@ -38,47 +36,19 @@ type Builder interface {
// Service runs the image side of deployments: pulls, update reconciliation, builds and dependents.
type Service struct {
- settingsService *settings.SettingsService
- imageService *image.ImageService
- dockerService *docker.DockerClientService
- buildService Builder
+ imageService *image.ImageService
+ dockerService *docker.DockerClientService
+ buildService Builder
}
-func New(settingsService *settings.SettingsService, imageService *image.ImageService, dockerService *docker.DockerClientService, buildService Builder) *Service {
+func New(imageService *image.ImageService, dockerService *docker.DockerClientService, buildService Builder) *Service {
return &Service{
- settingsService: settingsService,
- imageService: imageService,
- dockerService: dockerService,
- buildService: buildService,
+ imageService: imageService,
+ dockerService: dockerService,
+ buildService: buildService,
}
}
-// Pull pulls one image within the configured pull timeout and reconciles its update state.
-func (s *Service) Pull(
- ctx context.Context,
- imageRef string,
- progressWriter io.Writer,
- user usertypes.Actor,
- credentials []containerregistry.Credential,
-) error {
- cfg := s.settingsService.GetSettingsConfig()
-
- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(cfg.DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))
- defer pullCancel()
-
- if err := s.imageService.PullImage(pullCtx, imageRef, progressWriter, user, credentials); err != nil {
- if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {
- return fmt.Errorf("image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", imageRef)
- }
- return fmt.Errorf("failed to pull image %s: %w", imageRef, err)
- }
-
- if err := s.imageService.ReconcilePulledImageUpdate(ctx, imageRef); err != nil {
- slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", imageRef, "error", err)
- }
- return nil
-}
-
// NamespaceDependents splits dependents into running and stopped-only services; never-created ones are skipped.
// A ComposePs failure treats every dependent as running, since a dangling namespace reference is worse.
func (s *Service) NamespaceDependents(ctx context.Context, compProj *composetypes.Project, services []string) (running, stopped []string) {
@@ -114,8 +84,13 @@ func (s *Service) ImageOperations(user *usertypes.Actor, credentials []container
return projecttypes.ComposeImageOperations{
Exists: s.imageService.ImageExistsLocally,
LastTagged: s.imageService.ImageLastTagTime,
- Pull: func(ctx context.Context, image string, progress io.Writer) error {
- return s.Pull(ctx, image, progress, *cmp.Or(user, &usertypes.SystemUser), credentials)
+ Pull: func(ctx context.Context, ref string, progress io.Writer) error {
+ return s.imageService.Pull(ctx, image.PullRequest{
+ Ref: ref,
+ Progress: progress,
+ User: *cmp.Or(user, &usertypes.SystemUser),
+ Credentials: credentials,
+ })
},
Build: func(ctx context.Context, request buildtypes.BuildRequest, progress io.Writer, service string) error {
_, err := s.buildService.BuildImage(ctx, types.LocalDockerEnvironmentID, request, progress, service, user)
diff --git a/backend/internal/project/children/lifecycle/service.go b/backend/internal/project/children/lifecycle/service.go
index e97a5d2d1..a84ef4844 100644
--- a/backend/internal/project/children/lifecycle/service.go
+++ b/backend/internal/project/children/lifecycle/service.go
@@ -31,17 +31,14 @@ import (
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
- "github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
)
// Service runs pre-deploy scripts in throwaway runner containers.
type Service struct {
- settingsService *settings.SettingsService
- dockerService *dockerInternal.DockerClientService
- imageService *image.ImageService
+ dockerService *dockerInternal.DockerClientService
+ imageService *image.ImageService
}
const (
@@ -61,11 +58,10 @@ const (
// lifecycleEnvKeyRegex enforces POSIX-style env keys in configured Env and script stdout.
var lifecycleEnvKeyRegex = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
-func New(settingsService *settings.SettingsService, dockerService *dockerInternal.DockerClientService, imageService *image.ImageService) *Service {
+func New(dockerService *dockerInternal.DockerClientService, imageService *image.ImageService) *Service {
return &Service{
- settingsService: settingsService,
- dockerService: dockerService,
- imageService: imageService,
+ dockerService: dockerService,
+ imageService: imageService,
}
}
@@ -152,28 +148,22 @@ func (s *Service) RunScript(
timeout time.Duration,
actor user.Actor,
) (stdoutContent, stderrContent string, exitCode int64, err error) {
- dockerClient, dErr := s.dockerService.GetClient(ctx)
- if dErr != nil {
- return "", "", 0, fmt.Errorf("failed to connect to Docker: %w", dErr)
- }
-
// Pull a missing runner image through the image service so credentials and the pull timeout apply.
- if _, inspectErr := dockerClient.ImageInspect(ctx, runnerImage); inspectErr != nil {
- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))
- pullErr := s.imageService.PullImage(pullCtx, runnerImage, io.Discard, actor, nil)
- pullTimedOut := errors.Is(pullCtx.Err(), context.DeadlineExceeded)
- pullCancel()
- if pullErr != nil && pullTimedOut {
- return "", "", 0, fmt.Errorf("failed to ensure runner image %s: runner image pull timed out for %s (increase dockerImagePullTimeout setting if needed)", runnerImage, runnerImage)
- }
- if pullErr != nil {
- return "", "", 0, fmt.Errorf("failed to ensure runner image %s: pull runner image %s: %w", runnerImage, runnerImage, pullErr)
+ if _, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, runnerImage)
+ }); inspectErr != nil {
+ if pullErr := s.imageService.Pull(ctx, image.PullRequest{
+ Ref: runnerImage,
+ Progress: io.Discard,
+ User: actor,
+ }); pullErr != nil {
+ return "", "", 0, fmt.Errorf("failed to ensure runner image %s: %w", runnerImage, pullErr)
}
}
// MountForSubpath mounts a volume-backed /app/data via VolumeOptions.Subpath (host binds fail on e.g. WSL2).
// On the host, or when the mount cannot be derived, projectPath is bound as-is.
- inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ inspect, inspectErr := s.dockerService.Call(ctx, libarcane.InspectCurrentArcaneContainer)
if inspectErr != nil {
slog.WarnContext(ctx, "failed to derive workspace mount; falling back to bind on project path", "projectPath", projectPath, "error", inspectErr)
}
@@ -195,51 +185,54 @@ func (s *Service) RunScript(
env = append(env, k+"="+v)
}
- createCtx, createCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
- defer createCancel()
- resp, err := dockerClient.ContainerCreate(createCtx, client.ContainerCreateOptions{
- // Cmd is the script alone so its shebang picks the interpreter; Entrypoint is cleared for images like getsops/sops.
- Config: &container.Config{
- Image: runnerImage,
- Entrypoint: []string{},
- Cmd: []string{filepath.ToSlash(filepath.Join(lifecycleWorkspaceMount, scriptPath))},
- WorkingDir: lifecycleWorkspaceMount,
- Env: env,
- AttachStdout: true,
- AttachStderr: true,
- Labels: map[string]string{
- libarcane.InternalResourceLabel: "true",
+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ // Cmd is the script alone so its shebang picks the interpreter; Entrypoint is cleared for images like getsops/sops.
+ Config: &container.Config{
+ Image: runnerImage,
+ Entrypoint: []string{},
+ Cmd: []string{filepath.ToSlash(filepath.Join(lifecycleWorkspaceMount, scriptPath))},
+ WorkingDir: lifecycleWorkspaceMount,
+ Env: env,
+ AttachStdout: true,
+ AttachStderr: true,
+ Labels: map[string]string{
+ libarcane.InternalResourceLabel: "true",
+ },
+ },
+ // no-new-privileges and CapDrop ALL strip setuid and root capabilities from the script and its children.
+ HostConfig: &container.HostConfig{
+ Mounts: mounts,
+ NetworkMode: container.NetworkMode(cmp.Or(strings.TrimSpace(networkMode), "none")),
+ SecurityOpt: []string{"no-new-privileges:true"},
+ CapDrop: []string{"ALL"},
},
- },
- // no-new-privileges and CapDrop ALL strip setuid and root capabilities from the script and its children.
- HostConfig: &container.HostConfig{
- Mounts: mounts,
- NetworkMode: container.NetworkMode(cmp.Or(strings.TrimSpace(networkMode), "none")),
- SecurityOpt: []string{"no-new-privileges:true"},
- CapDrop: []string{"ALL"},
- },
+ })
})
if err != nil {
return "", "", 0, fmt.Errorf("create lifecycle container: %w", err)
}
defer func() {
// Cleanup outlives cancellation so a cancelled run never leaks its container.
- cleanupCtx, cleanupCancel := context.WithTimeout(
- context.WithoutCancel(ctx),
- timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI),
- )
- defer cleanupCancel()
- if _, removeErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, client.ContainerRemoveOptions{Force: true}); removeErr != nil && !errdefs.IsNotFound(removeErr) {
+ cleanupCtx := context.WithoutCancel(ctx)
+ if _, removeErr := s.dockerService.Call(cleanupCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ }); removeErr != nil && !errdefs.IsNotFound(removeErr) {
slog.WarnContext(cleanupCtx, "failed to remove lifecycle container", "containerId", resp.ID, "error", removeErr)
}
}()
- startCtx, startCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
- defer startCancel()
- if _, containerStartErr := dockerClient.ContainerStart(startCtx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
return "", "", 0, fmt.Errorf("start lifecycle container: %w", containerStartErr)
}
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return "", "", 0, fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+
logsCtx, logsCancel := context.WithCancel(ctx)
defer logsCancel()
logs, err := dockerClient.ContainerLogs(logsCtx, resp.ID, client.ContainerLogsOptions{
diff --git a/backend/internal/project/service.go b/backend/internal/project/service.go
index f3eaf6fd8..cbb817b09 100644
--- a/backend/internal/project/service.go
+++ b/backend/internal/project/service.go
@@ -20,6 +20,7 @@ import (
"time"
"github.com/compose-spec/compose-go/v2/types"
+ "github.com/containerd/errdefs"
"github.com/getarcaneapp/arcane/types/v2/base"
"github.com/getarcaneapp/arcane/types/v2/containerregistry"
imagetypes "github.com/getarcaneapp/arcane/types/v2/image"
@@ -83,6 +84,8 @@ const (
// inferredServiceCountBatchSize keeps one inferred count update under
// SQLite's bound variable limit (three placeholders per id).
inferredServiceCountBatchSize = 200
+ // leaseWaitTimeout bounds how long a project operation waits for one already running on the same project.
+ leaseWaitTimeout = 30 * time.Second
)
var (
@@ -110,6 +113,8 @@ type ProjectService struct {
// syncMu serializes SyncProjectsFromFileSystem: its discovery walk and its
// cleanup pass must not interleave with another run's.
syncMu sync.Mutex
+ // opLock serializes mutating operations on the same project.
+ opLock utils.KeyedMutex
composeNames composeNameCache
parsedCompose projecttypes.ComposeCache[*types.Project]
@@ -358,7 +363,7 @@ func NewProjectService(
return buildUpdateInfoSummary(imageRefs, projectdetails.MergeProjectContainerUpdateInfo(nil, services, scoped), nil)
},
),
- deployment: deployment.New(settingsService, imageService, dockerService, buildService),
+ deployment: deployment.New(imageService, dockerService, buildService),
// Rename recovery compares the stored name and path against its journal and rolls the row back.
updates: update.New(
kvService,
@@ -396,6 +401,50 @@ func (s *ProjectService) RecoverProjectRenameJournals(ctx context.Context) error
return s.updates.RecoverAll(ctx)
}
+// RecoverStuckProjectStatuses re-derives the status of projects left Deploying or Restarting by a restart.
+func (s *ProjectService) RecoverStuckProjectStatuses(ctx context.Context) error {
+ var stuck []Project
+ if err := s.db.WithContext(ctx).Where("status IN ?", []ProjectStatus{ProjectStatusDeploying, ProjectStatusRestarting}).Find(&stuck).Error; err != nil {
+ return fmt.Errorf("list projects with interrupted status: %w", err)
+ }
+
+ var firstErr error
+ recovered := 0
+ for _, proj := range stuck {
+ err := func() error {
+ unlock, err := s.lockProject(ctx, "recover project status", proj.ID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
+ services, err := s.projectServices(ctx, proj.ID)
+ if err != nil {
+ return fmt.Errorf("load services for project %s: %w", proj.ID, err)
+ }
+ status := ProjectStatus(listing.ProjectStatus(services))
+ if status == ProjectStatusUnknown {
+ slog.WarnContext(ctx, "Leaving interrupted project status unchanged: runtime status unknown", "projectId", proj.ID, "status", proj.Status)
+ return nil
+ }
+ if updateErr := s.updateProjectStatusAndCounts(ctx, proj.ID, status); updateErr != nil {
+ return updateErr
+ }
+ recovered++
+ return nil
+ }()
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to recover interrupted project status", "projectId", proj.ID, "error", err)
+ firstErr = cmp.Or(firstErr, err)
+ }
+ }
+
+ if len(stuck) > 0 {
+ slog.InfoContext(ctx, "Recovered interrupted project statuses", "found", len(stuck), "recovered", recovered)
+ }
+ return firstErr
+}
+
func (s *ProjectService) ResolveRegistryCredentials(ctx context.Context) ([]containerregistry.Credential, error) {
credentials, err := s.RegistryCredentialsProvider(ctx)
if err != nil {
@@ -425,6 +474,17 @@ func (s *ProjectService) getMutableProject(ctx context.Context, projectID string
return proj, nil
}
+// lockProject takes the project's operation lease, waiting up to leaseWaitTimeout for an operation already running on it.
+func (s *ProjectService) lockProject(ctx context.Context, op, projectID string) (func(), error) {
+ waitCtx, cancel := context.WithTimeout(ctx, leaseWaitTimeout)
+ defer cancel()
+ unlock, err := s.opLock.LockContext(waitCtx, "0:project:"+projectID)
+ if err != nil {
+ return nil, fmt.Errorf("%s %s: waiting for in-progress operation: %w", op, projectID, err)
+ }
+ return unlock, nil
+}
+
func (s *ProjectService) logProjectEvent(ctx context.Context, eventType event.EventType, projectID, projectName string, user usertypes.Actor, metadata database.JSON, action string) {
if logErr := s.eventService.LogProjectEvent(ctx, eventType, projectID, projectName, user.ID, user.Username, "0", metadata); logErr != nil {
slog.ErrorContext(ctx, "could not log project event", "action", action, "error", logErr)
@@ -797,7 +857,7 @@ func NewLifecycleService(
db: db,
settingsService: settingsService,
eventService: eventService,
- hooks: lifecycle.New(settingsService, dockerService, imageService),
+ hooks: lifecycle.New(dockerService, imageService),
}
}
@@ -983,6 +1043,12 @@ func ParseExtraMountsText(raw *string) ([]lifecycletype.ExtraMount, error) {
}
func (s *ProjectService) UpdateProjectServices(ctx context.Context, projectID string, servicesToUpdate []string, user usertypes.Actor, discoverTags bool) error {
+ unlock, err := s.lockProject(ctx, "update project services", projectID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
proj, err := s.getMutableProject(ctx, projectID)
if err != nil {
return err
@@ -1048,25 +1114,7 @@ func (s *ProjectService) UpdateProjectServices(ctx context.Context, projectID st
}
},
Recover: func(ctx context.Context) {
- services, servicesErr := s.projectServices(ctx, projectID)
- if servicesErr != nil {
- slog.WarnContext(ctx, "failed to inspect project services after deploy failure", "projectId", projectID, "error", servicesErr)
- } else {
- serviceCount, runningCount := listing.ServiceCounts(services)
- updateErr := s.db.WithContext(ctx).Model(&Project{}).Where("id = ?", projectID).Updates(map[string]any{
- "status": ProjectStatus(listing.ProjectStatus(services)),
- "service_count": serviceCount,
- "running_count": runningCount,
- "updated_at": time.Now(),
- }).Error
- if updateErr == nil {
- return
- }
- slog.WarnContext(ctx, "failed to restore project status after deploy failure", "projectId", projectID, "error", updateErr)
- }
- if updateErr := s.updateProjectStatus(ctx, projectID, ProjectStatusStopped); updateErr != nil {
- slog.WarnContext(ctx, "failed to set stopped status after deploy failure", "projectId", projectID, "error", updateErr)
- }
+ s.restoreRuntimeProjectStatus(ctx, projectID)
},
}); updateServicesErr != nil {
return updateServicesErr
@@ -1159,6 +1207,15 @@ func (s *ProjectService) UnarchiveProject(ctx context.Context, projectID string,
}
func (s *ProjectService) DeployProject(ctx context.Context, projectID string, user usertypes.Actor, options *projecttypes.DeployOptions) error {
+ unlock, err := s.lockProject(ctx, "deploy project", projectID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+ return s.deployProject(ctx, projectID, user, options)
+}
+
+func (s *ProjectService) deployProject(ctx context.Context, projectID string, user usertypes.Actor, options *projecttypes.DeployOptions) error {
projectFromDb, err := s.getMutableProject(ctx, projectID)
switch {
case errors.Is(err, common.ErrProjectArchived):
@@ -1215,25 +1272,7 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us
}, resolveRegistryCredentialsErr
},
Recover: func(ctx context.Context) {
- services, servicesErr := s.projectServices(ctx, projectID)
- if servicesErr != nil {
- slog.WarnContext(ctx, "failed to inspect project services after deploy failure", "projectId", projectID, "error", servicesErr)
- } else {
- serviceCount, runningCount := listing.ServiceCounts(services)
- updateErr := s.db.WithContext(ctx).Model(&Project{}).Where("id = ?", projectID).Updates(map[string]any{
- "status": ProjectStatus(listing.ProjectStatus(services)),
- "service_count": serviceCount,
- "running_count": runningCount,
- "updated_at": time.Now(),
- }).Error
- if updateErr == nil {
- return
- }
- slog.WarnContext(ctx, "failed to restore project status after deploy failure", "projectId", projectID, "error", updateErr)
- }
- if updateErr := s.updateProjectStatus(ctx, projectID, ProjectStatusStopped); updateErr != nil {
- slog.WarnContext(ctx, "failed to set stopped status after deploy failure", "projectId", projectID, "error", updateErr)
- }
+ s.restoreRuntimeProjectStatus(ctx, projectID)
},
})
if err != nil {
@@ -1254,14 +1293,24 @@ func (s *ProjectService) DeployProject(ctx context.Context, projectID string, us
}
func (s *ProjectService) DownProject(ctx context.Context, projectID string, user usertypes.Actor) error {
+ unlock, err := s.lockProject(ctx, "down project", projectID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
projectFromDb, err := s.getMutableProject(ctx, projectID)
if err != nil {
return err
}
+ return s.downProject(ctx, projectFromDb, user)
+}
+// downProject brings the project down and restores its previous status on failure.
+func (s *ProjectService) downProject(ctx context.Context, projectFromDb *Project, user usertypes.Actor) error {
+ projectID := projectFromDb.ID
proj, _, lerr := s.loadComposeProjectForProject(ctx, projectFromDb, nil)
if lerr != nil {
- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)
return fmt.Errorf("failed to load compose project: %w", lerr)
}
@@ -1272,7 +1321,7 @@ func (s *ProjectService) DownProject(ctx context.Context, projectID string, user
defer s.eventService.BeginComposeSuppressionWindow(proj.Name)()
if composeDownErr := projects.ComposeDown(ctx, proj, false); composeDownErr != nil {
- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)
+ _ = s.updateProjectStatus(ctx, projectID, projectFromDb.Status)
return fmt.Errorf("failed to bring down project: %w", composeDownErr)
}
@@ -1478,6 +1527,12 @@ func (s *ProjectService) DestroyProject(ctx context.Context, projectID string, r
"userId", user.ID,
"username", user.Username)
+ unlock, err := s.lockProject(ctx, "destroy project", projectID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
proj, err := s.GetProjectFromDatabaseByID(ctx, projectID)
if err != nil {
return err
@@ -1487,18 +1542,25 @@ func (s *ProjectService) DestroyProject(ctx context.Context, projectID string, r
"projectName", proj.Name,
"projectPath", proj.Path)
- if downProjectErr := s.DownProject(ctx, projectID, usertypes.SystemUser); downProjectErr != nil {
- slog.WarnContext(ctx, "failed to bring down project", "error", downProjectErr)
+ // Teardown runs down, volumes, files, then the DB row, stopping at the first real failure.
+ // Missing Docker resources and a missing compose file leave nothing to tear down.
+ missing := func(err error) bool {
+ return errdefs.IsNotFound(err) || errors.Is(err, common.ErrProjectComposeFileNotFound)
+ }
+ if downErr := s.downProject(ctx, proj, usertypes.SystemUser); downErr != nil && !missing(downErr) {
+ return fmt.Errorf("failed to bring down project: %w", downErr)
}
if removeVolumes {
- if compProj, _, lerr := s.loadComposeProjectForProject(ctx, proj, nil); lerr == nil {
+ compProj, _, lerr := s.loadComposeProjectForProject(ctx, proj, nil)
+ switch {
+ case lerr == nil:
defer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()
- if derr := projects.ComposeDown(ctx, compProj, true); derr != nil {
- slog.WarnContext(ctx, "failed to remove volumes", "error", derr)
+ if derr := projects.ComposeDown(ctx, compProj, true); derr != nil && !missing(derr) {
+ return fmt.Errorf("failed to remove project volumes: %w", derr)
}
- } else {
- slog.WarnContext(ctx, "failed to load compose project for volume removal", "error", lerr)
+ case !missing(lerr):
+ return fmt.Errorf("failed to load compose project for volume removal: %w", lerr)
}
}
@@ -1546,6 +1608,12 @@ func (s *ProjectService) DestroyProject(ctx context.Context, projectID string, r
}
func (s *ProjectService) RedeployProject(ctx context.Context, projectID string, user usertypes.Actor, options *projecttypes.DeployOptions) error {
+ unlock, err := s.lockProject(ctx, "redeploy project", projectID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
proj, err := s.getMutableProject(ctx, projectID)
if err != nil {
return err
@@ -1578,7 +1646,7 @@ func (s *ProjectService) RedeployProject(ctx context.Context, projectID string,
slog.WarnContext(ctx, "failed to pull project images", "error", pullProjectImagesErr)
}
- return s.DeployProject(ctx, projectID, user, options)
+ return s.deployProject(ctx, projectID, user, options)
}
func (s *ProjectService) PullProjectImages(ctx context.Context, projectID string, progressWriter io.Writer, user usertypes.Actor, credentials []containerregistry.Credential) error {
@@ -1595,7 +1663,12 @@ func (s *ProjectService) PullProjectImages(ctx context.Context, projectID string
defer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()
for _, imageRef := range projects.PullableImageRefs(compProj) {
- if pullErr := s.deployment.Pull(ctx, imageRef, progressWriter, user, credentials); pullErr != nil {
+ if pullErr := s.imageService.Pull(ctx, image.PullRequest{
+ Ref: imageRef,
+ Progress: progressWriter,
+ User: user,
+ Credentials: credentials,
+ }); pullErr != nil {
return pullErr
}
}
@@ -1619,6 +1692,12 @@ func (s *ProjectService) BuildProjectServices(ctx context.Context, projectID str
}
func (s *ProjectService) RestartProject(ctx context.Context, projectID string, services []string, user usertypes.Actor) error {
+ unlock, err := s.lockProject(ctx, "restart project", projectID)
+ if err != nil {
+ return err
+ }
+ defer unlock()
+
proj, err := s.getMutableProject(ctx, projectID)
if err != nil {
return err
@@ -1627,20 +1706,19 @@ func (s *ProjectService) RestartProject(ctx context.Context, projectID string, s
return resolveProjectComposeFileErr
}
- if updateProjectStatusErr := s.updateProjectStatus(ctx, projectID, ProjectStatusRestarting); updateProjectStatusErr != nil {
- return fmt.Errorf("failed to update project status to restarting: %w", updateProjectStatusErr)
- }
-
compProj, _, lerr := s.loadComposeProjectForProject(ctx, proj, nil)
if lerr != nil {
- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)
return fmt.Errorf("failed to load compose project: %w", lerr)
}
+ if updateProjectStatusErr := s.updateProjectStatus(ctx, projectID, ProjectStatusRestarting); updateProjectStatusErr != nil {
+ return fmt.Errorf("failed to update project status to restarting: %w", updateProjectStatusErr)
+ }
+
defer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()
if composeRestartErr := projects.ComposeRestart(ctx, compProj, services); composeRestartErr != nil {
- _ = s.updateProjectStatus(ctx, projectID, ProjectStatusRunning)
+ _ = s.updateProjectStatus(ctx, projectID, proj.Status)
return fmt.Errorf("failed to restart project: %w", composeRestartErr)
}
@@ -1676,6 +1754,20 @@ func (s *ProjectService) updateProjectStatusAndCounts(ctx context.Context, proje
return nil
}
+// restoreRuntimeProjectStatus re-derives status from the running services after a failed deploy, falling back to Stopped.
+func (s *ProjectService) restoreRuntimeProjectStatus(ctx context.Context, projectID string) {
+ services, err := s.projectServices(ctx, projectID)
+ if err == nil {
+ if err = s.updateProjectStatusAndCounts(ctx, projectID, ProjectStatus(listing.ProjectStatus(services))); err == nil {
+ return
+ }
+ }
+ slog.WarnContext(ctx, "failed to restore project status after deploy failure", "projectId", projectID, "error", err)
+ if updateErr := s.updateProjectStatus(ctx, projectID, ProjectStatusStopped); updateErr != nil {
+ slog.WarnContext(ctx, "failed to set stopped status after deploy failure", "projectId", projectID, "error", updateErr)
+ }
+}
+
func (s *ProjectService) updateProjectStatus(ctx context.Context, id string, status ProjectStatus) error {
if err := s.db.WithContext(ctx).Model(&Project{}).Where("id = ?", id).Updates(map[string]any{
"status": status,
@@ -2734,6 +2826,12 @@ func (s *ProjectService) reconcileComposeTagsForProject(ctx context.Context, pro
}
func (s *ProjectService) UpdateProject(ctx context.Context, projectID string, name, composeContent, envContent, overrideContent *string, user usertypes.Actor) (*Project, error) {
+ unlock, err := s.lockProject(ctx, "update project", projectID)
+ if err != nil {
+ return nil, err
+ }
+ defer unlock()
+
recoverErr := s.updates.RecoverProject(ctx, projectID)
proj, err := s.getMutableProject(ctx, projectID)
if err != nil {
@@ -2945,6 +3043,12 @@ func (s *ProjectService) ApplyGitSyncProjectFiles(
gitOverrideFileName string,
user usertypes.Actor,
) (*Project, bool, error) {
+ unlock, err := s.lockProject(ctx, "apply git sync project files", projectID)
+ if err != nil {
+ return nil, false, err
+ }
+ defer unlock()
+
proj, err := s.getMutableProject(ctx, projectID)
if err != nil {
return nil, false, err
@@ -3202,6 +3306,12 @@ func (s *ProjectService) UpdateProjectWorkspace(
if err := workspacepkg.ValidateUpdateManifest(manifest.FileTreeRevision, len(manifest.FileChanges), 500); err != nil {
return nil, common.Classify(common.ErrProjectWorkspaceBadRequest, err)
}
+ unlock, err := s.lockProject(ctx, "update project workspace", projectID)
+ if err != nil {
+ return nil, err
+ }
+ defer unlock()
+
proj, err := s.getMutableProject(ctx, projectID)
if err != nil {
return nil, err
diff --git a/backend/internal/project/service_test.go b/backend/internal/project/service_test.go
index 385c3fee8..619d0b656 100644
--- a/backend/internal/project/service_test.go
+++ b/backend/internal/project/service_test.go
@@ -417,6 +417,93 @@ func TestProjectService_DestroyProject_PreservesFilesWhenRequested(t *testing.T)
assert.FileExists(t, filepath.Join(projectsDir, trashName, "project-data.txt"))
}
+func TestProjectService_LockProject_SerializesSameProject(t *testing.T) {
+ svc, _, _ := setupProjectDestroyTestService(t)
+
+ unlock, err := svc.lockProject(t.Context(), "test", "project-a")
+ require.NoError(t, err)
+
+ done := make(chan error, 1)
+ go func() { done <- svc.DownProject(t.Context(), "project-a", usertypes.Actor{}) }()
+
+ select {
+ case downErr := <-done:
+ t.Fatalf("DownProject ran while another operation held the project lease: %v", downErr)
+ case <-time.After(100 * time.Millisecond):
+ }
+
+ otherCtx, cancel := context.WithTimeout(t.Context(), time.Second)
+ defer cancel()
+ otherErr := svc.DownProject(otherCtx, "project-b", usertypes.Actor{})
+ require.Error(t, otherErr)
+ require.NotErrorIs(t, otherErr, context.DeadlineExceeded, "a different project must not wait on the held lease")
+
+ unlock()
+ select {
+ case downErr := <-done:
+ require.Error(t, downErr)
+ require.NotErrorIs(t, downErr, context.DeadlineExceeded)
+ case <-time.After(2 * time.Second):
+ t.Fatal("DownProject did not run after the project lease was released")
+ }
+}
+
+func TestProjectService_LockProject_ReturnsWaitErrorOnDeadline(t *testing.T) {
+ svc, _, _ := setupProjectDestroyTestService(t)
+
+ unlock, err := svc.lockProject(t.Context(), "test", "project-a")
+ require.NoError(t, err)
+ defer unlock()
+
+ ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+ defer cancel()
+ downErr := svc.DownProject(ctx, "project-a", usertypes.Actor{})
+ require.ErrorIs(t, downErr, context.DeadlineExceeded)
+ assert.Contains(t, downErr.Error(), "down project project-a: waiting for in-progress operation")
+}
+
+func TestProjectService_DownProject_RestoresPreviousStatusOnFailure(t *testing.T) {
+ ctx := t.Context()
+ svc, db, projectsDir := setupProjectDestroyTestService(t)
+ t.Setenv("DOCKER_HOST", "tcp://127.0.0.1:1")
+
+ require.NoError(t, db.Create(&Project{
+ ID: "project-down-fails",
+ Name: "down-fails",
+ DirName: new("down-fails"),
+ Path: createComposeProjectDir(t, projectsDir, "down-fails"),
+ Status: ProjectStatusStopped,
+ }).Error)
+
+ require.Error(t, svc.DownProject(ctx, "project-down-fails", usertypes.Actor{}))
+
+ var stored Project
+ require.NoError(t, db.First(&stored, "id = ?", "project-down-fails").Error)
+ assert.Equal(t, ProjectStatusStopped, stored.Status)
+}
+
+func TestProjectService_DestroyProject_KeepsRowWhenDownFails(t *testing.T) {
+ ctx := t.Context()
+ svc, db, projectsDir := setupProjectDestroyTestService(t)
+ t.Setenv("DOCKER_HOST", "tcp://127.0.0.1:1")
+
+ projectPath := createComposeProjectDir(t, projectsDir, "destroy-down-fails")
+ require.NoError(t, db.Create(&Project{
+ ID: "project-destroy-down-fails",
+ Name: "destroy-down-fails",
+ DirName: new("destroy-down-fails"),
+ Path: projectPath,
+ Status: ProjectStatusRunning,
+ }).Error)
+
+ require.Error(t, svc.DestroyProject(ctx, "project-destroy-down-fails", true, true, usertypes.Actor{}))
+
+ var stored Project
+ require.NoError(t, db.First(&stored, "id = ?", "project-destroy-down-fails").Error)
+ assert.Equal(t, ProjectStatusRunning, stored.Status)
+ assert.DirExists(t, projectPath)
+}
+
func TestProjectService_GetProjectFromDatabaseByID(t *testing.T) {
db := setupProjectTestDB(t)
ctx := t.Context()
@@ -701,7 +788,7 @@ func TestProjectService_PullProjectImages_UpdatesCurrentImageRecordAfterPull(t *
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)
svc := NewProjectService(
db,
settingsService,
@@ -809,7 +896,7 @@ func TestProjectService_EnsureImagesPresent_UpdatesCurrentImageRecordAfterPull(t
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)
svc := NewProjectService(
db,
settingsService,
@@ -889,7 +976,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(t
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)
svc := NewProjectService(
db,
settingsService,
@@ -914,7 +1001,7 @@ func TestProjectService_PullImageForService_UpdatesCurrentImageRecordAfterPull(t
CheckTime: time.Now().UTC().Add(-time.Hour),
}).Error)
- require.NoError(t, svc.deployment.Pull(ctx, imageRef, io.Discard, usertypes.SystemUser, nil))
+ require.NoError(t, svc.deployment.ImageOperations(nil, nil).Pull(ctx, imageRef, io.Discard))
// sha256:old-worker may still be in use by another container — must not be cleared (fixes #2453).
var oldRecord imageupdate.ImageUpdateRecord
@@ -964,7 +1051,7 @@ func TestProjectService_ComposePullSelectedServices_ReconcilesOnlyOnSuccess(t *t
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}
eventService := event.NewEventService(db, nil, nil)
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)
svc := NewProjectService(
db,
settingsService,
@@ -1080,7 +1167,7 @@ func TestProjectService_ComposePullSelectedServices_LeavesRecordsWhenPullFails(t
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, failingServer)}
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), settingsService, nil)
svc := NewProjectService(
db,
settingsService,
@@ -1150,7 +1237,7 @@ func TestProjectService_UpdateProjectServicesHardFailsWhenPullFails(t *testing.T
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, failingServer)}
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, event.NewEventService(db, nil, nil), settingsService, nil)
projectPath := createComposeProjectDir(t, projectsDir, "compose-update-pull-fail")
require.NoError(
@@ -1250,7 +1337,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreate(t *testing.T) {
dockerService := &docker.DockerClientService{Client: newTestDockerClient(t, server)}
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
eventService := event.NewEventService(db, nil, nil)
- imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, nil, nil)
+ imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService, settingsService, nil)
projectPath := createComposeProjectDir(t, projectsDir, "compose-update-force")
require.NoError(
@@ -2155,6 +2242,85 @@ func TestProjectService_UpdateProject_RenameRejectsStaleStoppedWhenRuntimeIsRunn
assert.Equal(t, ProjectStatusStopped, fromDB.Status)
}
+func TestProjectService_RecoverStuckProjectStatuses(t *testing.T) {
+ tests := []struct {
+ name string
+ saved ProjectStatus
+ containers []container.Summary
+ want ProjectStatus
+ }{
+ {
+ name: "deploying with running containers becomes running",
+ saved: ProjectStatusDeploying,
+ containers: []container.Summary{
+ {
+ ID: "app-container",
+ Names: []string{"/foo-app-1"},
+ Image: "nginx:alpine",
+ State: container.StateRunning,
+ Status: "Up 30 seconds",
+ Labels: map[string]string{
+ api.ProjectLabel: "foo",
+ api.ServiceLabel: "app",
+ },
+ },
+ },
+ want: ProjectStatusRunning,
+ },
+ {
+ // Compose services without a container are reported as stopped.
+ name: "restarting without containers becomes stopped",
+ saved: ProjectStatusRestarting,
+ want: ProjectStatusStopped,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx := t.Context()
+
+ projectsDir := t.TempDir()
+ t.Setenv("PROJECTS_DIRECTORY", projectsDir)
+
+ settingsService, err := newSettingsServiceForTest(t, ctx, db)
+ require.NoError(t, err)
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
+
+ server := configureProjectRuntimeDocker(t, tt.containers)
+ svc := NewProjectService(
+ db,
+ settingsService,
+ event.NewEventService(db, nil, nil),
+ image.NewImageService(db, nil, nil, nil, nil, nil, nil, nil),
+ &docker.DockerClientService{Client: newTestDockerClient(t, server)},
+ testBuildBuilder{},
+ nil,
+ nil,
+ config.Load(),
+ kv.NewKVService(db),
+ testRegistryCredentials,
+ )
+
+ dirName := "foo"
+ project := &Project{
+ ID: "proj-stuck",
+ Name: "foo",
+ DirName: &dirName,
+ Path: createComposeProjectDir(t, projectsDir, dirName),
+ Status: tt.saved,
+ }
+ require.NoError(t, db.Create(project).Error)
+
+ require.NoError(t, svc.RecoverStuckProjectStatuses(ctx))
+
+ var fromDB Project
+ require.NoError(t, db.First(&fromDB, "id = ?", project.ID).Error)
+ assert.Equal(t, tt.want, fromDB.Status)
+ })
+ }
+}
+
func TestProjectService_UpdateProject_RenameResolvesUnknownStoppedStatusBeforeVolumeMigration(t *testing.T) {
db := setupProjectTestDB(t)
ctx := t.Context()
@@ -8670,7 +8836,7 @@ func TestUpdateProjectServiceImagesPersistsBeforeDeploymentAndRetries(t *testing
db,
settingsService,
event.NewEventService(db, nil, nil),
- image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), nil, nil),
+ image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), settingsService, nil),
testDockerService,
testBuildBuilder{},
nil,
@@ -9058,7 +9224,7 @@ func TestProjectServiceManualUpdateDiscoversTags(t *testing.T) {
db,
settingsService,
event.NewEventService(db, nil, nil),
- image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), nil, nil),
+ image.NewImageService(db, pullDocker, nil, imageupdate.NewImageUpdateService(db, nil, nil, pullDocker, nil, nil, nil), nil, event.NewEventService(db, nil, nil), settingsService, nil),
testDockerService,
testBuildBuilder{},
nil,
diff --git a/backend/internal/swarm/children/configs/service.go b/backend/internal/swarm/children/configs/service.go
index a6b5b02d0..af7dcdae8 100644
--- a/backend/internal/swarm/children/configs/service.go
+++ b/backend/internal/swarm/children/configs/service.go
@@ -2,7 +2,6 @@ package configs
import (
"context"
- stdjson "encoding/json"
"encoding/json/v2"
"errors"
"fmt"
@@ -11,16 +10,18 @@ import (
swarmtypes "github.com/getarcaneapp/arcane/types/v2/swarm"
"github.com/moby/moby/api/types/swarm"
"github.com/moby/moby/client"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
)
// Service manages swarm configs.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
+ docker *docker.DockerClientService
ensureManager func(ctx context.Context) error
}
-func NewService(dockerClient func(ctx context.Context) (*client.Client, error), ensureManager func(ctx context.Context) error) *Service {
- return &Service{dockerClient: dockerClient, ensureManager: ensureManager}
+func NewService(dockerService *docker.DockerClientService, ensureManager func(ctx context.Context) error) *Service {
+ return &Service{docker: dockerService, ensureManager: ensureManager}
}
func (s *Service) ListConfigs(ctx context.Context) ([]swarmtypes.ConfigSummary, error) {
@@ -28,12 +29,9 @@ func (s *Service) ListConfigs(ctx context.Context) ([]swarmtypes.ConfigSummary,
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- configsResult, err := dockerClient.ConfigList(ctx, client.ConfigListOptions{})
+ configsResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigListResult, error) {
+ return dockerClient.ConfigList(ctx, client.ConfigListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list swarm configs: %w", err)
}
@@ -50,12 +48,9 @@ func (s *Service) GetConfig(ctx context.Context, configID string) (*swarmtypes.C
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- cfgResult, err := dockerClient.ConfigInspect(ctx, configID, client.ConfigInspectOptions{})
+ cfgResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigInspectResult, error) {
+ return dockerClient.ConfigInspect(ctx, configID, client.ConfigInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm config: %w", err)
}
@@ -68,17 +63,20 @@ func (s *Service) CreateConfig(ctx context.Context, req swarmtypes.ConfigCreateR
return nil, err
}
- spec, err := decodeConfigSpecInternal(req.Spec)
- if err != nil {
- return nil, err
+ if trimmed := strings.TrimSpace(string(req.Spec)); trimmed == "" || trimmed == "null" {
+ return nil, errors.New("config spec is required")
}
-
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ var spec swarm.ConfigSpec
+ if err := json.Unmarshal(req.Spec, &spec); err != nil {
+ return nil, fmt.Errorf("failed to parse config spec: %w", err)
+ }
+ if strings.TrimSpace(spec.Name) == "" {
+ return nil, errors.New("config spec name is required")
}
- createResult, err := dockerClient.ConfigCreate(ctx, client.ConfigCreateOptions{Spec: spec})
+ createResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigCreateResult, error) {
+ return dockerClient.ConfigCreate(ctx, client.ConfigCreateOptions{Spec: spec})
+ })
if err != nil {
return nil, fmt.Errorf("failed to create swarm config: %w", err)
}
@@ -91,31 +89,11 @@ func (s *Service) RemoveConfig(ctx context.Context, configID string) error {
return err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, configRemoveErr := dockerClient.ConfigRemove(ctx, configID, client.ConfigRemoveOptions{}); configRemoveErr != nil {
+ if _, configRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ConfigRemoveResult, error) {
+ return dockerClient.ConfigRemove(ctx, configID, client.ConfigRemoveOptions{})
+ }); configRemoveErr != nil {
return fmt.Errorf("failed to remove swarm config: %w", configRemoveErr)
}
return nil
}
-
-func decodeConfigSpecInternal(raw stdjson.RawMessage) (swarm.ConfigSpec, error) {
- if len(raw) == 0 || strings.TrimSpace(string(raw)) == "" || strings.TrimSpace(string(raw)) == "null" {
- return swarm.ConfigSpec{}, errors.New("config spec is required")
- }
-
- var spec swarm.ConfigSpec
- if err := json.Unmarshal(raw, &spec); err != nil {
- return swarm.ConfigSpec{}, fmt.Errorf("failed to parse config spec: %w", err)
- }
-
- if strings.TrimSpace(spec.Name) == "" {
- return swarm.ConfigSpec{}, errors.New("config spec name is required")
- }
-
- return spec, nil
-}
diff --git a/backend/internal/swarm/children/nodes/service.go b/backend/internal/swarm/children/nodes/service.go
index eb2dc3f51..f67c30215 100644
--- a/backend/internal/swarm/children/nodes/service.go
+++ b/backend/internal/swarm/children/nodes/service.go
@@ -9,6 +9,7 @@ import (
"fmt"
"log/slog"
"net/http"
+ "slices"
"strings"
"sync"
"time"
@@ -17,10 +18,12 @@ import (
"github.com/moby/moby/api/types/swarm"
"github.com/moby/moby/client"
"github.com/samber/hot"
+ "github.com/samber/mo"
"go.getarcane.app/kit/pkg"
"golang.org/x/sync/errgroup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
@@ -35,7 +38,7 @@ const (
// Service manages swarm nodes, their agent bindings, and Easy Join.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
+ docker *dockerInternal.DockerClientService
ensureManager func(ctx context.Context) error
listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error)
localJoinTokens func(ctx context.Context) (*swarmtypes.SwarmJoinTokensResponse, error)
@@ -44,14 +47,14 @@ type Service struct {
}
func NewService(
- dockerClient func(ctx context.Context) (*client.Client, error),
+ dockerService *dockerInternal.DockerClientService,
ensureManager func(ctx context.Context) error,
listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error),
localJoinTokens func(ctx context.Context) (*swarmtypes.SwarmJoinTokensResponse, error),
environmentService *environment.EnvironmentService,
) *Service {
return &Service{
- dockerClient: dockerClient,
+ docker: dockerService,
ensureManager: ensureManager,
listTasks: listTasks,
localJoinTokens: localJoinTokens,
@@ -84,7 +87,7 @@ type swarmNodeAgentCoverage struct {
candidatesByNodeID map[string][]environment.Environment
localIdentity *SwarmNodeIdentity
// Resolved once per request rather than per node: the name is user-editable, and
- // applyNodeAgentCoverageInternal runs in a loop with no context to look it up.
+ // applyNodeAgentCoverage runs in a loop with no context to look it up.
localEnvironmentName string
}
@@ -101,8 +104,8 @@ func (s *Service) ListNodesPaginated(ctx context.Context, environmentID string,
return nil, pagination.Response{}, errors.New("remote swarm node listing failed")
}
- s.enrichNodeAgentStatusesInternal(ctx, environmentID, remote.Data)
- result := s.buildNodePaginationConfigInternal().SearchOrderAndPaginate(remote.Data, params)
+ s.enrichNodeAgentStatuses(ctx, environmentID, remote.Data)
+ result := s.buildNodePaginationConfig().SearchOrderAndPaginate(remote.Data, params)
return result.Items, pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params), nil
}
@@ -110,12 +113,9 @@ func (s *Service) ListNodesPaginated(ctx context.Context, environmentID string,
return nil, pagination.Response{}, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})
+ nodesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {
+ return dockerClient.NodeList(ctx, client.NodeListOptions{})
+ })
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list swarm nodes: %w", err)
}
@@ -126,9 +126,9 @@ func (s *Service) ListNodesPaginated(ctx context.Context, environmentID string,
items = append(items, swarmtypes.NewNodeSummary(node))
}
- s.enrichNodeAgentStatusesInternal(ctx, environmentID, items)
+ s.enrichNodeAgentStatuses(ctx, environmentID, items)
- config := s.buildNodePaginationConfigInternal()
+ config := s.buildNodePaginationConfig()
result := config.SearchOrderAndPaginate(items, params)
paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params)
@@ -149,7 +149,7 @@ func (s *Service) GetNode(ctx context.Context, environmentID, nodeID string) (*s
return nil, errors.New("remote swarm node inspection failed")
}
items := []swarmtypes.NodeSummary{remote.Data}
- s.enrichNodeAgentStatusesInternal(ctx, environmentID, items)
+ s.enrichNodeAgentStatuses(ctx, environmentID, items)
return &items[0], nil
}
@@ -157,18 +157,15 @@ func (s *Service) GetNode(ctx context.Context, environmentID, nodeID string) (*s
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- nodeResult, err := dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})
+ nodeResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeInspectResult, error) {
+ return dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm node: %w", err)
}
items := []swarmtypes.NodeSummary{swarmtypes.NewNodeSummary(nodeResult.Node)}
- s.enrichNodeAgentStatusesInternal(ctx, environmentID, items)
+ s.enrichNodeAgentStatuses(ctx, environmentID, items)
return new(items[0]), nil
}
@@ -201,10 +198,11 @@ func (s *Service) ReconcileNodeAgents(ctx context.Context, environmentID string)
continue
}
- result.State = swarmtypes.NodeAgentStateConnected
- result.EnvironmentID = &bound.ID
- result.Candidates = nil
- results = append(results, result)
+ results = append(results, swarmtypes.NodeAgentReconcileResult{
+ NodeID: node.ID,
+ State: swarmtypes.NodeAgentStateConnected,
+ EnvironmentID: &bound.ID,
+ })
}
return &swarmtypes.NodeAgentReconcileResponse{Results: results}, nil
@@ -221,7 +219,7 @@ func (s *Service) BindNodeAgent(ctx context.Context, parentEnvironmentID, nodeID
return nil, errors.New("only visible environments can be attached")
}
- runtime := s.resolveSwarmNodeAgentRuntimeInternal(ctx, localEnvironment)
+ runtime := s.resolveSwarmNodeAgentRuntime(ctx, localEnvironment)
if runtime.identity == nil || !runtime.identity.SwarmActive {
return nil, errors.New("environment did not report an active swarm identity")
}
@@ -237,12 +235,9 @@ func (s *Service) BindNodeAgent(ctx context.Context, parentEnvironmentID, nodeID
}
func (s *Service) GetLocalNodeIdentity(ctx context.Context) (*SwarmNodeIdentity, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})
+ infoResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SystemInfoResult, error) {
+ return dockerClient.Info(ctx, client.InfoOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect local Docker engine: %w", err)
}
@@ -263,11 +258,11 @@ func (s *Service) GetLocalNodeIdentity(ctx context.Context) (*SwarmNodeIdentity,
}, nil
}
-func (s *Service) enrichNodeAgentStatusesInternal(ctx context.Context, environmentID string, items []swarmtypes.NodeSummary) {
+func (s *Service) enrichNodeAgentStatuses(ctx context.Context, environmentID string, items []swarmtypes.NodeSummary) {
if s.environmentService == nil || len(items) == 0 || strings.TrimSpace(environmentID) == "" {
return
}
- coverage, err := s.resolveNodeAgentCoverageInternal(ctx, environmentID)
+ coverage, err := s.resolveNodeAgentCoverage(ctx, environmentID)
if err != nil {
slog.WarnContext(ctx, "failed to resolve swarm node agent coverage", "environmentId", environmentID, "error", err.Error())
return
@@ -275,11 +270,11 @@ func (s *Service) enrichNodeAgentStatusesInternal(ctx context.Context, environme
for index := range items {
item := &items[index]
- s.applyNodeAgentCoverageInternal(environmentID, item, coverage)
+ s.applyNodeAgentCoverage(environmentID, item, coverage)
}
}
-func (s *Service) resolveNodeAgentCoverageInternal(ctx context.Context, environmentID string) (*swarmNodeAgentCoverage, error) {
+func (s *Service) resolveNodeAgentCoverage(ctx context.Context, environmentID string) (*swarmNodeAgentCoverage, error) {
agentEnvs, err := s.environmentService.ListSwarmNodeAgentEnvironments(ctx, environmentID)
if err != nil {
return nil, err
@@ -308,7 +303,7 @@ func (s *Service) resolveNodeAgentCoverageInternal(ctx context.Context, environm
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "swarm worker")
- runtime := s.resolveSwarmNodeAgentRuntimeInternal(groupCtx, &env)
+ runtime := s.resolveSwarmNodeAgentRuntime(groupCtx, &env)
runtimeMu.Lock()
runtimeByEnvID[env.ID] = runtime
runtimeMu.Unlock()
@@ -354,99 +349,60 @@ func (s *Service) resolveNodeAgentCoverageInternal(ctx context.Context, environm
return coverage, nil
}
-func (s *Service) applyNodeAgentCoverageInternal(environmentID string, item *swarmtypes.NodeSummary, coverage *swarmNodeAgentCoverage) {
+func (s *Service) applyNodeAgentCoverage(environmentID string, item *swarmtypes.NodeSummary, coverage *swarmNodeAgentCoverage) {
if item == nil || coverage == nil {
return
}
nodeID := strings.TrimSpace(item.ID)
if environmentID == environment.LocalEnvironmentID && coverage.localIdentity != nil && coverage.localIdentity.SwarmActive && strings.TrimSpace(coverage.localIdentity.SwarmNodeID) == nodeID {
- connected := true
- bindingKind := swarmtypes.NodeAgentBindingKindLocal
- localID, localType := environment.LocalEnvironmentID, "local"
- localName := environment.DisplayName(environment.LocalEnvironmentID, coverage.localEnvironmentName)
item.Agent = swarmtypes.NodeAgentStatus{
State: swarmtypes.NodeAgentStateConnected,
- BindingKind: &bindingKind,
- EnvironmentID: &localID,
- EnvironmentName: &localName,
- EnvironmentType: &localType,
- Connected: &connected,
+ BindingKind: new(swarmtypes.NodeAgentBindingKindLocal),
+ EnvironmentID: new(environment.LocalEnvironmentID),
+ EnvironmentName: new(environment.DisplayName(environment.LocalEnvironmentID, coverage.localEnvironmentName)),
+ EnvironmentType: new("local"),
+ Connected: new(true),
}
return
}
boundEnvs := coverage.boundEnvsByNodeID[nodeID]
if len(boundEnvs) > 0 {
- localEnvironment := preferredNodeAgentEnvironmentInternal(boundEnvs)
- item.Agent = s.buildNodeAgentStatusInternal(nodeID, &localEnvironment, coverage.runtimeByEnvID[localEnvironment.ID])
+ // Prefer the first visible environment, falling back to the first one.
+ localEnvironment := boundEnvs[max(0, slices.IndexFunc(boundEnvs, func(env environment.Environment) bool { return !env.Hidden }))]
+ item.Agent = s.buildNodeAgentStatus(nodeID, &localEnvironment, coverage.runtimeByEnvID[localEnvironment.ID])
return
}
candidates := coverage.candidatesByNodeID[nodeID]
if len(candidates) > 1 {
- item.Agent = swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateAmbiguous, Candidates: buildNodeAgentCandidatesInternal(candidates)}
+ item.Agent = swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateAmbiguous, Candidates: buildNodeAgentCandidates(candidates)}
return
}
if len(candidates) == 1 {
- item.Agent.Candidates = buildNodeAgentCandidatesInternal(candidates)
+ item.Agent.Candidates = buildNodeAgentCandidates(candidates)
}
}
-func (s *Service) resolveSwarmNodeAgentRuntimeInternal(ctx context.Context, env *environment.Environment) swarmNodeAgentRuntime {
+func (s *Service) resolveSwarmNodeAgentRuntime(ctx context.Context, env *environment.Environment) swarmNodeAgentRuntime {
if env == nil {
return swarmNodeAgentRuntime{}
}
- runtime := swarmNodeAgentRuntime{
- connected: edge.HasActiveTunnel(env.ID),
- }
-
- if tunnelState, ok := edge.GetTunnelRuntimeState(env.ID).Get(); ok {
- runtime.lastHeartbeat = tunnelState.LastHeartbeat
- }
-
- if pollState, ok := edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).Get(); ok {
- runtime.lastPollAt = pollState.LastPollAt
- }
-
- if identity := s.cachedSwarmNodeIdentityInternal(env.ID); identity != nil {
- runtime.connected = true
- runtime.identity = identity
- return runtime
- }
-
- identity, err := s.fetchSwarmNodeIdentityViaEdgeInternal(ctx, env.ID)
- if err != nil {
- slog.DebugContext(ctx, "failed to probe swarm node identity", "environmentId", env.ID, "error", err.Error())
- return runtime
- }
-
- runtime.connected = true
- runtime.identity = identity
- s.cacheSwarmNodeIdentityInternal(env.ID, *identity)
- return runtime
-}
-
-func (s *Service) cachedSwarmNodeIdentityInternal(environmentID string) *SwarmNodeIdentity {
- if s.identityCache == nil {
- return nil
- }
- identity, ok, _ := s.identityCache.Get(environmentID)
- if !ok {
- return nil
- }
- return &identity
-}
-
-func (s *Service) cacheSwarmNodeIdentityInternal(environmentID string, identity SwarmNodeIdentity) {
- if s.identityCache != nil {
- s.identityCache.Set(environmentID, identity)
+ identity, found, _ := s.identityCache.Get(env.ID)
+ if !found {
+ if fetched, err := s.fetchSwarmNodeIdentityViaEdge(ctx, env.ID); err != nil {
+ slog.DebugContext(ctx, "failed to probe swarm node identity", "environmentId", env.ID, "error", err.Error())
+ } else {
+ identity, found = *fetched, true
+ s.identityCache.Set(env.ID, identity)
+ }
}
-}
-
-func (s *Service) invalidateSwarmNodeIdentityInternal(environmentID string) {
- if s.identityCache != nil {
- s.identityCache.Delete(environmentID)
+ return swarmNodeAgentRuntime{
+ connected: found || edge.HasActiveTunnel(env.ID),
+ lastHeartbeat: kit.FromPtr(edge.GetTunnelRuntimeState(env.ID).OrEmpty()).LastHeartbeat,
+ lastPollAt: edge.GetPollRuntimeRegistry().Get(env.ID, time.Now()).OrEmpty().LastPollAt,
+ identity: kit.Ternary(found, &identity, nil),
}
}
@@ -461,11 +417,11 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID str
if err != nil {
return nil, err
}
- remoteAddrs, err := selectSwarmManagerAddressesInternal(request.RemoteAddrs, nodes)
+ remoteAddrs, err := selectSwarmManagerAddresses(request.RemoteAddrs, nodes)
if err != nil {
return nil, err
}
- tokens, err := s.getSwarmJoinTokensForEnvironmentInternal(ctx, managerEnvironmentID)
+ tokens, err := s.getSwarmJoinTokensForEnvironment(ctx, managerEnvironmentID)
if err != nil {
return nil, err
}
@@ -475,14 +431,14 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID str
}
results := make([]swarmtypes.SwarmJoinEnvironmentResult, len(request.Targets))
- processTargetInternal := func(index int) {
+ processTarget := func(index int) {
target := request.Targets[index]
- results[index] = s.joinEnvironmentInternal(ctx, managerEnvironmentID, target, remoteAddrs, tokens, memberNodeIDs)
+ results[index] = s.joinEnvironment(ctx, managerEnvironmentID, target, remoteAddrs, tokens, memberNodeIDs)
}
for index, target := range request.Targets {
if target.Role == swarmtypes.SwarmJoinEnvironmentRoleManager {
- processTargetInternal(index)
+ processTarget(index)
}
}
@@ -496,7 +452,7 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID str
defer utils.RecoverToError(&workerErr, "swarm worker")
if groupCtx.Err() == nil {
- processTargetInternal(index)
+ processTarget(index)
}
return nil
})
@@ -506,7 +462,7 @@ func (s *Service) JoinEnvironments(ctx context.Context, managerEnvironmentID str
return &swarmtypes.SwarmJoinEnvironmentsResponse{Results: results}, nil
}
-func (s *Service) getSwarmJoinTokensForEnvironmentInternal(ctx context.Context, environmentID string) (*swarmtypes.SwarmJoinTokensResponse, error) {
+func (s *Service) getSwarmJoinTokensForEnvironment(ctx context.Context, environmentID string) (*swarmtypes.SwarmJoinTokensResponse, error) {
if environmentID == "0" {
return s.localJoinTokens(ctx)
}
@@ -524,7 +480,7 @@ func (s *Service) getSwarmJoinTokensForEnvironmentInternal(ctx context.Context,
return &response.Data, nil
}
-func (s *Service) joinEnvironmentInternal(
+func (s *Service) joinEnvironment(
ctx context.Context,
managerEnvironmentID string,
target swarmtypes.SwarmJoinEnvironmentTarget,
@@ -532,97 +488,83 @@ func (s *Service) joinEnvironmentInternal(
tokens *swarmtypes.SwarmJoinTokensResponse,
memberNodeIDs map[string]struct{},
) swarmtypes.SwarmJoinEnvironmentResult {
- result := swarmtypes.SwarmJoinEnvironmentResult{EnvironmentID: target.EnvironmentID, State: swarmtypes.SwarmJoinEnvironmentResultFailed}
- localEnvironment, err := s.environmentService.GetEnvironmentByID(ctx, target.EnvironmentID)
- if err != nil || localEnvironment.Hidden || !localEnvironment.Enabled {
- message := "target environment is unavailable"
- if err != nil {
- message = err.Error()
+ result := func(state swarmtypes.SwarmJoinEnvironmentResultState, nodeID, message string) swarmtypes.SwarmJoinEnvironmentResult {
+ return swarmtypes.SwarmJoinEnvironmentResult{
+ EnvironmentID: target.EnvironmentID,
+ State: state,
+ NodeID: kit.Ternary(nodeID == "", nil, &nodeID),
+ Error: kit.Ternary(message == "", nil, &message),
}
- result.Error = &message
- return result
+ }
+ failed := func(message string) swarmtypes.SwarmJoinEnvironmentResult {
+ return result(swarmtypes.SwarmJoinEnvironmentResultFailed, "", message)
}
- runtime := s.resolveSwarmNodeAgentRuntimeInternal(ctx, localEnvironment)
+ localEnvironment, err := s.environmentService.GetEnvironmentByID(ctx, target.EnvironmentID)
+ if err != nil {
+ return failed(err.Error())
+ }
+ if localEnvironment.Hidden || !localEnvironment.Enabled {
+ return failed("target environment is unavailable")
+ }
+
+ runtime := s.resolveSwarmNodeAgentRuntime(ctx, localEnvironment)
if runtime.identity != nil && runtime.identity.SwarmActive {
nodeID := strings.TrimSpace(runtime.identity.SwarmNodeID)
if _, member := memberNodeIDs[nodeID]; !member {
- message := "environment is active in another swarm cluster"
- result.Error = &message
- return result
+ return failed("environment is active in another swarm cluster")
}
- if _, bindSwarmNodeEnvironmentErr := s.environmentService.BindSwarmNodeEnvironment(ctx, managerEnvironmentID, nodeID, target.EnvironmentID, true); bindSwarmNodeEnvironmentErr != nil {
- message := bindSwarmNodeEnvironmentErr.Error()
- result.Error = &message
- return result
+ if _, bindErr := s.environmentService.BindSwarmNodeEnvironment(ctx, managerEnvironmentID, nodeID, target.EnvironmentID, true); bindErr != nil {
+ return failed(bindErr.Error())
}
- result.State = swarmtypes.SwarmJoinEnvironmentResultAlreadyMember
- result.NodeID = &nodeID
- return result
+ return result(swarmtypes.SwarmJoinEnvironmentResultAlreadyMember, nodeID, "")
}
- joinToken := tokens.Worker
- if target.Role == swarmtypes.SwarmJoinEnvironmentRoleManager {
- joinToken = tokens.Manager
- }
- joinRequest := swarmtypes.SwarmJoinRequest{
+ joinToken := kit.Ternary(target.Role == swarmtypes.SwarmJoinEnvironmentRoleManager, tokens.Manager, tokens.Worker)
+ body, err := json.Marshal(swarmtypes.SwarmJoinRequest{
ListenAddr: target.ListenAddr,
AdvertiseAddr: target.AdvertiseAddr,
DataPathAddr: target.DataPathAddr,
RemoteAddrs: remoteAddrs,
JoinToken: joinToken,
Availability: target.Availability,
- }
- body, err := json.Marshal(joinRequest)
+ })
if err != nil {
- message := err.Error()
- result.Error = &message
- return result
+ return failed(err.Error())
}
var joinResponse struct {
Success bool `json:"success"`
}
- if proxyJSONRequestErr := s.environmentService.ProxyJSONRequest(ctx, target.EnvironmentID, http.MethodPost, "/api/environments/0/swarm/join", body, &joinResponse); proxyJSONRequestErr != nil {
- message := describeSwarmJoinFailureInternal(proxyJSONRequestErr, joinToken)
- result.Error = &message
- return result
+ if proxyErr := s.environmentService.ProxyJSONRequest(ctx, target.EnvironmentID, http.MethodPost, "/api/environments/0/swarm/join", body, &joinResponse); proxyErr != nil {
+ return failed(describeSwarmJoinFailure(proxyErr, joinToken))
}
if !joinResponse.Success {
- message := "swarm join failed"
- result.Error = &message
- return result
+ return failed("swarm join failed")
}
for range 6 {
- s.invalidateSwarmNodeIdentityInternal(target.EnvironmentID)
- verifiedRuntime := s.resolveSwarmNodeAgentRuntimeInternal(ctx, localEnvironment)
+ s.identityCache.Delete(target.EnvironmentID)
+ verifiedRuntime := s.resolveSwarmNodeAgentRuntime(ctx, localEnvironment)
if verifiedRuntime.identity != nil && verifiedRuntime.identity.SwarmActive {
nodeID := strings.TrimSpace(verifiedRuntime.identity.SwarmNodeID)
if _, getNodeErr := s.GetNode(ctx, managerEnvironmentID, nodeID); getNodeErr == nil {
if _, bindErr := s.environmentService.BindSwarmNodeEnvironment(ctx, managerEnvironmentID, nodeID, target.EnvironmentID, true); bindErr == nil {
- result.State = swarmtypes.SwarmJoinEnvironmentResultJoined
- result.NodeID = &nodeID
- result.Error = nil
- return result
+ return result(swarmtypes.SwarmJoinEnvironmentResultJoined, nodeID, "")
}
}
}
select {
case <-ctx.Done():
- message := ctx.Err().Error()
- result.Error = &message
- return result
+ return failed(ctx.Err().Error())
case <-time.After(time.Second):
}
}
- result.State = swarmtypes.SwarmJoinEnvironmentResultJoinedUnverified
- result.Error = nil
- return result
+ return result(swarmtypes.SwarmJoinEnvironmentResultJoinedUnverified, "", "")
}
-func (s *Service) fetchSwarmNodeIdentityViaEdgeInternal(ctx context.Context, environmentID string) (*SwarmNodeIdentity, error) {
+func (s *Service) fetchSwarmNodeIdentityViaEdge(ctx context.Context, environmentID string) (*SwarmNodeIdentity, error) {
reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
@@ -640,69 +582,45 @@ func (s *Service) fetchSwarmNodeIdentityViaEdgeInternal(ctx context.Context, env
return parsed.Data, nil
}
-func (s *Service) buildNodeAgentStatusInternal(nodeID string, env *environment.Environment, runtime swarmNodeAgentRuntime) swarmtypes.NodeAgentStatus {
+func (s *Service) buildNodeAgentStatus(nodeID string, env *environment.Environment, runtime swarmNodeAgentRuntime) swarmtypes.NodeAgentStatus {
if env == nil {
return swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateNone}
}
- // An agent is live when it holds an open tunnel or has checked in via the poll
- // control plane recently. lastHeartbeat/lastPollAt are only populated from the
- // in-memory runtime registries when activity is fresh, so their presence is
- // current evidence — unlike env.Status, which poll-mode agents never advance
- // out of "pending" (HandlePoll updates the registry, not the persisted record).
+ // Live means an open tunnel or a fresh poll check-in; the runtime registries only hold fresh activity,
+ // unlike env.Status, which poll-mode agents never advance out of "pending".
live := runtime.connected || runtime.lastHeartbeat != nil || runtime.lastPollAt != nil
-
- status := swarmtypes.NodeAgentStatus{
- State: swarmtypes.NodeAgentStateOffline,
- EnvironmentID: &env.ID,
- Connected: &live,
- LastHeartbeat: runtime.lastHeartbeat,
- LastPollAt: runtime.lastPollAt,
-
- EnvironmentName: &env.Name,
- }
- environmentType := kit.Ternary(env.IsEdge, "edge", "direct")
- status.EnvironmentType = &environmentType
- bindingKind := kit.Ternary(env.Hidden, swarmtypes.NodeAgentBindingKindDedicated, swarmtypes.NodeAgentBindingKindEnvironment)
- status.BindingKind = &bindingKind
-
- if runtime.identity != nil {
- if reportedNodeID := strings.TrimSpace(runtime.identity.SwarmNodeID); reportedNodeID != "" {
- status.ReportedNodeID = &reportedNodeID
- }
- if reportedHostname := strings.TrimSpace(runtime.identity.Hostname); reportedHostname != "" {
- status.ReportedHostname = &reportedHostname
- }
- }
-
- // A live tunnel with a completed identity probe lets us authoritatively detect
- // a node-ID mismatch (only tunnel mode can probe identity).
- if runtime.connected && runtime.identity != nil {
- if !runtime.identity.SwarmActive || strings.TrimSpace(runtime.identity.SwarmNodeID) != strings.TrimSpace(nodeID) {
- status.State = swarmtypes.NodeAgentStateMismatched
- return status
- }
- status.State = swarmtypes.NodeAgentStateConnected
- return status
- }
-
- // Poll-mode agents (and tunnels without a completed probe) can't be identity-
- // verified live, but a fresh check-in proves the agent mapped to this node is
- // reachable. The node-to-env mapping was established by Arcane at deploy time,
- // so report it as connected instead of masking it behind a stale env.Status.
- if live {
- status.State = swarmtypes.NodeAgentStateConnected
- return status
- }
-
- // No live evidence: an unpaired agent stays pending; one seen before is offline.
- if env.Status == string(environment.EnvironmentStatusPending) || env.LastSeen == nil {
- status.State = swarmtypes.NodeAgentStatePending
- return status
+ identity := kit.FromPtr(runtime.identity)
+ reportedNodeID := strings.TrimSpace(identity.SwarmNodeID)
+ reportedHostname := strings.TrimSpace(identity.Hostname)
+ state := func() swarmtypes.NodeAgentState {
+ switch {
+ // Only a live tunnel with a completed identity probe can authoritatively detect a node-ID mismatch.
+ case runtime.connected && runtime.identity != nil:
+ return kit.Ternary(identity.SwarmActive && reportedNodeID == strings.TrimSpace(nodeID), swarmtypes.NodeAgentStateConnected, swarmtypes.NodeAgentStateMismatched)
+ // A fresh check-in proves the agent Arcane mapped to this node at deploy time is reachable.
+ case live:
+ return swarmtypes.NodeAgentStateConnected
+ // No live evidence: an unpaired agent stays pending; one seen before is offline.
+ case env.Status == string(environment.EnvironmentStatusPending) || env.LastSeen == nil:
+ return swarmtypes.NodeAgentStatePending
+ default:
+ return swarmtypes.NodeAgentStateOffline
+ }
+ }()
+
+ return swarmtypes.NodeAgentStatus{
+ State: state,
+ BindingKind: new(kit.Ternary(env.Hidden, swarmtypes.NodeAgentBindingKindDedicated, swarmtypes.NodeAgentBindingKindEnvironment)),
+ EnvironmentID: &env.ID,
+ EnvironmentName: &env.Name,
+ EnvironmentType: new(kit.Ternary(env.IsEdge, "edge", "direct")),
+ Connected: &live,
+ LastHeartbeat: runtime.lastHeartbeat,
+ LastPollAt: runtime.lastPollAt,
+ ReportedNodeID: kit.Ternary(reportedNodeID == "", nil, &reportedNodeID),
+ ReportedHostname: kit.Ternary(reportedHostname == "", nil, &reportedHostname),
}
-
- status.State = swarmtypes.NodeAgentStateOffline
- return status
}
func (s *Service) UpdateNode(ctx context.Context, nodeID string, req swarmtypes.NodeUpdateRequest) error {
@@ -710,35 +628,28 @@ func (s *Service) UpdateNode(ctx context.Context, nodeID string, req swarmtypes.
return err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- nodeResult, err := dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})
+ nodeResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeInspectResult, error) {
+ return dockerClient.NodeInspect(ctx, nodeID, client.NodeInspectOptions{})
+ })
if err != nil {
return fmt.Errorf("failed to inspect swarm node: %w", err)
}
version := cmp.Or(req.Version, nodeResult.Node.Version.Index)
- spec := nodeResult.Node.Spec
- if req.Name != nil {
- spec.Name = *req.Name
- }
- if req.Labels != nil {
- spec.Labels = req.Labels
- }
- if req.Role != nil {
- spec.Role = *req.Role
- }
- if req.Availability != nil {
- spec.Availability = *req.Availability
+ current := nodeResult.Node.Spec
+ spec := swarm.NodeSpec{
+ Name: mo.PointerToOption(req.Name).OrElse(current.Name),
+ Labels: kit.Ternary(req.Labels != nil, req.Labels, current.Labels),
+ Role: mo.PointerToOption(req.Role).OrElse(current.Role),
+ Availability: mo.PointerToOption(req.Availability).OrElse(current.Availability),
}
- if _, nodeUpdateErr := dockerClient.NodeUpdate(ctx, nodeID, client.NodeUpdateOptions{
- Version: swarm.Version{Index: version},
- Spec: spec,
+ if _, nodeUpdateErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeUpdateResult, error) {
+ return dockerClient.NodeUpdate(ctx, nodeID, client.NodeUpdateOptions{
+ Version: swarm.Version{Index: version},
+ Spec: spec,
+ })
}); nodeUpdateErr != nil {
return fmt.Errorf("failed to update swarm node: %w", nodeUpdateErr)
}
@@ -751,12 +662,9 @@ func (s *Service) RemoveNode(ctx context.Context, nodeID string, force bool) err
return err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, nodeRemoveErr := dockerClient.NodeRemove(ctx, nodeID, client.NodeRemoveOptions{Force: force}); nodeRemoveErr != nil {
+ if _, nodeRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeRemoveResult, error) {
+ return dockerClient.NodeRemove(ctx, nodeID, client.NodeRemoveOptions{Force: force})
+ }); nodeRemoveErr != nil {
return fmt.Errorf("failed to remove swarm node: %w", nodeRemoveErr)
}
@@ -781,7 +689,7 @@ func (s *Service) ListNodeTasksPaginated(ctx context.Context, nodeID string, par
return s.listTasks(ctx, filters, params)
}
-func (s *Service) buildNodePaginationConfigInternal() pagination.Config[swarmtypes.NodeSummary] {
+func (s *Service) buildNodePaginationConfig() pagination.Config[swarmtypes.NodeSummary] {
return pagination.Config[swarmtypes.NodeSummary]{
SearchAccessors: []pagination.SearchAccessor[swarmtypes.NodeSummary]{
func(node swarmtypes.NodeSummary) (string, error) { return node.Hostname, nil },
@@ -808,7 +716,7 @@ func (s *Service) GetJoinCandidates(ctx context.Context, environmentID string) (
if environmentID == environment.LocalEnvironmentID {
identity, err = s.GetLocalNodeIdentity(ctx)
} else {
- identity, err = s.fetchSwarmNodeIdentityViaEdgeInternal(ctx, environmentID)
+ identity, err = s.fetchSwarmNodeIdentityViaEdge(ctx, environmentID)
}
if err != nil {
return nil, fmt.Errorf("failed to inspect Easy Join manager: %w", err)
@@ -854,10 +762,10 @@ func (s *Service) GetJoinCandidates(ctx context.Context, environmentID string) (
return candidates, nil
}
-// describeSwarmJoinFailureInternal turns a proxied join failure into a concise,
+// describeSwarmJoinFailure turns a proxied join failure into a concise,
// token-free message. Agent HTTP errors contribute their detail; transport
// errors keep their own text.
-func describeSwarmJoinFailureInternal(err error, joinToken string) string {
+func describeSwarmJoinFailure(err error, joinToken string) string {
var status *remenv.StatusError
if !errors.As(err, &status) {
return RedactSwarmJoinToken(err.Error(), joinToken)
@@ -877,7 +785,7 @@ func describeSwarmJoinFailureInternal(err error, joinToken string) string {
return fmt.Sprintf("swarm join failed with HTTP %d from the target agent", status.StatusCode)
}
-// redactSwarmJoinTokenInternal masks the join token wherever it appears in message.
+// redactSwarmJoinToken masks the join token wherever it appears in message.
func RedactSwarmJoinToken(message, joinToken string) string {
if joinToken == "" {
return message
@@ -885,9 +793,9 @@ func RedactSwarmJoinToken(message, joinToken string) string {
return strings.ReplaceAll(message, joinToken, "[redacted]")
}
-// selectSwarmManagerAddressesInternal prefers explicit manager addresses and
+// selectSwarmManagerAddresses prefers explicit manager addresses and
// otherwise uses the addresses advertised by the cluster's manager nodes.
-func selectSwarmManagerAddressesInternal(explicit []string, nodes []swarmtypes.NodeSummary) ([]string, error) {
+func selectSwarmManagerAddresses(explicit []string, nodes []swarmtypes.NodeSummary) ([]string, error) {
addrs := kit.TrimNonEmpty(explicit)
if len(addrs) > 0 {
return addrs, nil
@@ -903,7 +811,7 @@ func selectSwarmManagerAddressesInternal(explicit []string, nodes []swarmtypes.N
return addrs, nil
}
-func buildNodeAgentCandidatesInternal(environments []environment.Environment) []swarmtypes.NodeAgentCandidate {
+func buildNodeAgentCandidates(environments []environment.Environment) []swarmtypes.NodeAgentCandidate {
candidates := make([]swarmtypes.NodeAgentCandidate, 0, len(environments))
for _, environment := range environments {
environmentType := kit.Ternary(environment.IsEdge, "edge", "direct")
@@ -915,16 +823,3 @@ func buildNodeAgentCandidatesInternal(environments []environment.Environment) []
}
return candidates
}
-
-func preferredNodeAgentEnvironmentInternal(environments []environment.Environment) environment.Environment {
- var preferred environment.Environment
- for index, environment := range environments {
- if index == 0 {
- preferred = environment
- }
- if !environment.Hidden {
- return environment
- }
- }
- return preferred
-}
diff --git a/backend/internal/swarm/children/nodes/service_test.go b/backend/internal/swarm/children/nodes/service_test.go
index d3a1d95d8..bb884523b 100644
--- a/backend/internal/swarm/children/nodes/service_test.go
+++ b/backend/internal/swarm/children/nodes/service_test.go
@@ -21,7 +21,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/remenv"
)
-func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {
+func setupSwarmServiceTestDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -29,7 +29,7 @@ func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newSettingsServiceForSwarmTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
svc, err := settings.NewSettingsService(ctx, db)
if err == nil {
@@ -38,7 +38,7 @@ func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, d
return svc, err
}
-func createSwarmTestEnvironmentInternal(t *testing.T, db *database.DB, id, apiURL, status string, isEdge bool, accessToken *string) {
+func createSwarmTestEnvironment(t *testing.T, db *database.DB, id, apiURL, status string, isEdge bool, accessToken *string) {
t.Helper()
now := time.Now()
require.NoError(t, db.Create(&environment.Environment{
@@ -52,26 +52,26 @@ func createSwarmTestEnvironmentInternal(t *testing.T, db *database.DB, id, apiUR
}).Error)
}
-func TestSelectSwarmManagerAddressesInternal(t *testing.T) {
+func TestSelectSwarmManagerAddresses(t *testing.T) {
nodes := []swarm.NodeSummary{
{ID: "manager-1", ManagerAddress: "10.0.0.1:2377"},
{ID: "worker-1"},
{ID: "manager-2", ManagerAddress: "10.0.0.2:2377"},
}
- addrs, err := selectSwarmManagerAddressesInternal([]string{" 100.64.0.10:2377 ", "", "100.64.0.11:2377"}, nodes)
+ addrs, err := selectSwarmManagerAddresses([]string{" 100.64.0.10:2377 ", "", "100.64.0.11:2377"}, nodes)
require.NoError(t, err)
require.Equal(t, []string{"100.64.0.10:2377", "100.64.0.11:2377"}, addrs)
- addrs, err = selectSwarmManagerAddressesInternal([]string{" "}, nodes)
+ addrs, err = selectSwarmManagerAddresses([]string{" "}, nodes)
require.NoError(t, err)
require.Equal(t, []string{"10.0.0.1:2377", "10.0.0.2:2377"}, addrs)
- _, err = selectSwarmManagerAddressesInternal(nil, []swarm.NodeSummary{{ID: "worker-1"}})
+ _, err = selectSwarmManagerAddresses(nil, []swarm.NodeSummary{{ID: "worker-1"}})
require.ErrorIs(t, err, common.ErrBadRequest)
}
-func TestDescribeSwarmJoinFailureInternal(t *testing.T) {
+func TestDescribeSwarmJoinFailure(t *testing.T) {
const token = "SWMTKN-1-secret"
statusErr := func(code int, body string) error {
return errors.Join(&remenv.StatusError{StatusCode: code, Body: []byte(body)})
@@ -79,7 +79,7 @@ func TestDescribeSwarmJoinFailureInternal(t *testing.T) {
require.Equal(t,
"failed to join swarm: Timeout was reached before node joined using [redacted]",
- describeSwarmJoinFailureInternal(
+ describeSwarmJoinFailure(
statusErr(
500,
"{\"title\":\"Internal Server Error\",\"status\":500,\"detail\":\"failed to join swarm: Timeout was reached be"+
@@ -88,20 +88,20 @@ func TestDescribeSwarmJoinFailureInternal(t *testing.T) {
token,
),
)
- require.Equal(t, "legacy join failure", describeSwarmJoinFailureInternal(statusErr(400, `{"success":false,"error":"legacy join failure"}`), token))
- require.Equal(t, "swarm join failed with HTTP 502 from the target agent", describeSwarmJoinFailureInternal(statusErr(502, "<html>bad gateway</html>"), token))
- require.Equal(t, "swarm join failed with HTTP 404 from the target agent", describeSwarmJoinFailureInternal(statusErr(404, ""), token))
+ require.Equal(t, "legacy join failure", describeSwarmJoinFailure(statusErr(400, `{"success":false,"error":"legacy join failure"}`), token))
+ require.Equal(t, "swarm join failed with HTTP 502 from the target agent", describeSwarmJoinFailure(statusErr(502, "<html>bad gateway</html>"), token))
+ require.Equal(t, "swarm join failed with HTTP 404 from the target agent", describeSwarmJoinFailure(statusErr(404, ""), token))
require.Equal(t,
"failed to send request to environment edge: dial tcp 10.0.0.9:3552: connect: connection refused",
- describeSwarmJoinFailureInternal(errors.New("failed to send request to environment edge: dial tcp 10.0.0.9:3552: connect: connection refused"), token),
+ describeSwarmJoinFailure(errors.New("failed to send request to environment edge: dial tcp 10.0.0.9:3552: connect: connection refused"), token),
)
- require.Equal(t, "token [redacted] rejected", describeSwarmJoinFailureInternal(errors.New("token "+token+" rejected"), token))
+ require.Equal(t, "token [redacted] rejected", describeSwarmJoinFailure(errors.New("token "+token+" rejected"), token))
}
-func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToken(t *testing.T) {
+func TestService_FetchSwarmNodeIdentityViaEdge_UsesEnvironmentAccessToken(t *testing.T) {
ctx := t.Context()
- db := setupSwarmServiceTestDBInternal(t)
- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)
+ db := setupSwarmServiceTestDB(t)
+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)
require.NoError(t, err)
envSvc := environment.NewEnvironmentService(db, nil, nil, nil, settingsSvc, nil)
@@ -125,7 +125,7 @@ func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToke
}))
defer server.Close()
- createSwarmTestEnvironmentInternal(
+ createSwarmTestEnvironment(
t,
db,
"env-1",
@@ -137,7 +137,7 @@ func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToke
svc := NewService(nil, nil, nil, nil, envSvc)
- identity, err := svc.fetchSwarmNodeIdentityViaEdgeInternal(ctx, "env-1")
+ identity, err := svc.fetchSwarmNodeIdentityViaEdge(ctx, "env-1")
require.NoError(t, err)
require.NotNil(t, identity)
require.Equal(t, "node-1", identity.SwarmNodeID)
@@ -151,7 +151,7 @@ func TestService_FetchSwarmNodeIdentityViaEdgeInternal_UsesEnvironmentAccessToke
// The regression case is a poll-mode agent: its persisted env.Status never leaves
// "pending" (HandlePoll only updates the in-memory poll registry), so a fresh
// lastPollAt must still resolve to "connected" rather than "pending".
-func TestService_BuildNodeAgentStatusInternal(t *testing.T) {
+func TestService_BuildNodeAgentStatus(t *testing.T) {
const nodeID = "node-abc"
now := time.Now()
svc := &Service{}
@@ -196,7 +196,7 @@ func TestService_BuildNodeAgentStatusInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := svc.buildNodeAgentStatusInternal(nodeID, tt.env, tt.runtime)
+ got := svc.buildNodeAgentStatus(nodeID, tt.env, tt.runtime)
require.Equal(t, tt.want, got.State)
})
}
diff --git a/backend/internal/swarm/children/secrets/service.go b/backend/internal/swarm/children/secrets/service.go
index 24068afc5..ce8df0eeb 100644
--- a/backend/internal/swarm/children/secrets/service.go
+++ b/backend/internal/swarm/children/secrets/service.go
@@ -2,7 +2,6 @@ package secrets
import (
"context"
- stdjson "encoding/json"
"encoding/json/v2"
"errors"
"fmt"
@@ -11,16 +10,18 @@ import (
swarmtypes "github.com/getarcaneapp/arcane/types/v2/swarm"
"github.com/moby/moby/api/types/swarm"
"github.com/moby/moby/client"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
)
// Service manages swarm secrets.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
+ docker *docker.DockerClientService
ensureManager func(ctx context.Context) error
}
-func NewService(dockerClient func(ctx context.Context) (*client.Client, error), ensureManager func(ctx context.Context) error) *Service {
- return &Service{dockerClient: dockerClient, ensureManager: ensureManager}
+func NewService(dockerService *docker.DockerClientService, ensureManager func(ctx context.Context) error) *Service {
+ return &Service{docker: dockerService, ensureManager: ensureManager}
}
func (s *Service) ListSecrets(ctx context.Context) ([]swarmtypes.SecretSummary, error) {
@@ -28,12 +29,9 @@ func (s *Service) ListSecrets(ctx context.Context) ([]swarmtypes.SecretSummary,
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- secretsResult, err := dockerClient.SecretList(ctx, client.SecretListOptions{})
+ secretsResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretListResult, error) {
+ return dockerClient.SecretList(ctx, client.SecretListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list swarm secrets: %w", err)
}
@@ -50,12 +48,9 @@ func (s *Service) GetSecret(ctx context.Context, secretID string) (*swarmtypes.S
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- secretResult, err := dockerClient.SecretInspect(ctx, secretID, client.SecretInspectOptions{})
+ secretResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretInspectResult, error) {
+ return dockerClient.SecretInspect(ctx, secretID, client.SecretInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm secret: %w", err)
}
@@ -68,17 +63,20 @@ func (s *Service) CreateSecret(ctx context.Context, req swarmtypes.SecretCreateR
return nil, err
}
- spec, err := decodeSecretSpecInternal(req.Spec)
- if err != nil {
- return nil, err
+ if trimmed := strings.TrimSpace(string(req.Spec)); trimmed == "" || trimmed == "null" {
+ return nil, errors.New("secret spec is required")
}
-
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
+ var spec swarm.SecretSpec
+ if err := json.Unmarshal(req.Spec, &spec); err != nil {
+ return nil, fmt.Errorf("failed to parse secret spec: %w", err)
+ }
+ if strings.TrimSpace(spec.Name) == "" {
+ return nil, errors.New("secret spec name is required")
}
- createResult, err := dockerClient.SecretCreate(ctx, client.SecretCreateOptions{Spec: spec})
+ createResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretCreateResult, error) {
+ return dockerClient.SecretCreate(ctx, client.SecretCreateOptions{Spec: spec})
+ })
if err != nil {
return nil, fmt.Errorf("failed to create swarm secret: %w", err)
}
@@ -91,31 +89,11 @@ func (s *Service) RemoveSecret(ctx context.Context, secretID string) error {
return err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, secretRemoveErr := dockerClient.SecretRemove(ctx, secretID, client.SecretRemoveOptions{}); secretRemoveErr != nil {
+ if _, secretRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SecretRemoveResult, error) {
+ return dockerClient.SecretRemove(ctx, secretID, client.SecretRemoveOptions{})
+ }); secretRemoveErr != nil {
return fmt.Errorf("failed to remove swarm secret: %w", secretRemoveErr)
}
return nil
}
-
-func decodeSecretSpecInternal(raw stdjson.RawMessage) (swarm.SecretSpec, error) {
- if len(raw) == 0 || strings.TrimSpace(string(raw)) == "" || strings.TrimSpace(string(raw)) == "null" {
- return swarm.SecretSpec{}, errors.New("secret spec is required")
- }
-
- var spec swarm.SecretSpec
- if err := json.Unmarshal(raw, &spec); err != nil {
- return swarm.SecretSpec{}, fmt.Errorf("failed to parse secret spec: %w", err)
- }
-
- if strings.TrimSpace(spec.Name) == "" {
- return swarm.SecretSpec{}, errors.New("secret spec name is required")
- }
-
- return spec, nil
-}
diff --git a/backend/internal/swarm/children/services/service.go b/backend/internal/swarm/children/services/service.go
index d0c27f51c..862cc88ec 100644
--- a/backend/internal/swarm/children/services/service.go
+++ b/backend/internal/swarm/children/services/service.go
@@ -16,23 +16,25 @@ import (
"github.com/moby/moby/api/types/swarm"
"github.com/moby/moby/client"
"go.getarcane.app/docker"
+ "go.getarcane.app/kit/pkg"
+ dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
)
// Service manages swarm services.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
+ docker *dockerInternal.DockerClientService
ensureManager func(ctx context.Context) error
listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error)
}
func NewService(
- dockerClient func(ctx context.Context) (*client.Client, error),
+ dockerService *dockerInternal.DockerClientService,
ensureManager func(ctx context.Context) error,
listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error),
) *Service {
- return &Service{dockerClient: dockerClient, ensureManager: ensureManager, listTasks: listTasks}
+ return &Service{docker: dockerService, ensureManager: ensureManager, listTasks: listTasks}
}
func (s *Service) ListServicesPaginated(ctx context.Context, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error) {
@@ -40,32 +42,56 @@ func (s *Service) ListServicesPaginated(ctx context.Context, params pagination.Q
return nil, pagination.Response{}, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{Status: true})
+ servicesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {
+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{Status: true})
+ })
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list swarm services: %w", err)
}
- return s.PaginateSummaries(ctx, dockerClient, servicesResult.Items, params)
+ return s.PaginateSummaries(ctx, servicesResult.Items, params)
}
// PaginateSummaries summarizes services with their running nodes and networks, then paginates them.
func (s *Service) PaginateSummaries(
ctx context.Context,
- dockerClient *client.Client,
services []swarm.Service,
params pagination.QueryParams,
) ([]swarmtypes.ServiceSummary, pagination.Response, error) {
- summaries, err := s.summarizeServicesInternal(ctx, dockerClient, services)
+ summaries, err := s.summarizeServices(ctx, services)
if err != nil {
return nil, pagination.Response{}, err
}
- result := s.buildServicePaginationConfigInternal().SearchOrderAndPaginate(summaries, params)
+ config := pagination.Config[swarmtypes.ServiceSummary]{
+ SearchAccessors: []pagination.SearchAccessor[swarmtypes.ServiceSummary]{
+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Name, nil },
+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Image, nil },
+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.ID, nil },
+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.StackName, nil },
+ func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Mode, nil },
+ func(svc swarmtypes.ServiceSummary) (string, error) {
+ return strings.Join(svc.Networks, " "), nil
+ },
+ func(svc swarmtypes.ServiceSummary) (string, error) {
+ return strings.Join(svc.Nodes, " "), nil
+ },
+ },
+ SortBindings: []pagination.SortBinding[swarmtypes.ServiceSummary]{
+ {Key: "name", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Name, b.Name) }},
+ {Key: "image", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Image, b.Image) }},
+ {Key: "mode", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Mode, b.Mode) }},
+ {Key: "replicas", Fn: func(a, b swarmtypes.ServiceSummary) int {
+ if localCmp := cmp.Compare(a.Replicas, b.Replicas); localCmp != 0 {
+ return localCmp
+ }
+ return cmp.Compare(a.RunningReplicas, b.RunningReplicas)
+ }},
+ {Key: "created", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},
+ {Key: "updated", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},
+ },
+ }
+ result := config.SearchOrderAndPaginate(summaries, params)
return result.Items, pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params), nil
}
@@ -74,27 +100,26 @@ func (s *Service) GetService(ctx context.Context, serviceID string) (*swarmtypes
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- serviceResult, err := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ serviceResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {
+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm service: %w", err)
}
service := serviceResult.Service
inspect := swarmtypes.NewServiceInspect(service)
- inspect.Nodes = s.resolveServiceNodeNamesInternal(ctx, dockerClient, serviceID)
- inspect.NetworkDetails = s.enrichServiceNetworkDetailsInternal(ctx, dockerClient, service.Spec.TaskTemplate.Networks)
- inspect.Mounts = s.enrichServiceMountsInternal(ctx, dockerClient, service.Spec.TaskTemplate.ContainerSpec)
+ inspect.Nodes = s.resolveServiceNodeNames(ctx, serviceID)
+ inspect.NetworkDetails = s.enrichServiceNetworkDetails(ctx, service.Spec.TaskTemplate.Networks)
+ inspect.Mounts = s.enrichServiceMounts(ctx, service.Spec.TaskTemplate.ContainerSpec)
return &inspect, nil
}
-func (s *Service) resolveServiceNodeNamesInternal(ctx context.Context, dockerClient *client.Client, serviceID string) []string {
- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})
+func (s *Service) resolveServiceNodeNames(ctx context.Context, serviceID string) []string {
+ nodesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {
+ return dockerClient.NodeList(ctx, client.NodeListOptions{})
+ })
if err != nil {
return nil
}
@@ -104,7 +129,9 @@ func (s *Service) resolveServiceNodeNamesInternal(ctx context.Context, dockerCli
nodeNameByID[node.ID] = node.Description.Hostname
}
- tasksResult, err := dockerClient.TaskList(ctx, client.TaskListOptions{})
+ tasksResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {
+ return dockerClient.TaskList(ctx, client.TaskListOptions{})
+ })
if err != nil {
return nil
}
@@ -123,9 +150,8 @@ func (s *Service) resolveServiceNodeNamesInternal(ctx context.Context, dockerCli
return nodeNames
}
-func (s *Service) enrichServiceNetworkDetailsInternal(
+func (s *Service) enrichServiceNetworkDetails(
ctx context.Context,
- dockerClient *client.Client,
networkConfigs []swarm.NetworkAttachmentConfig,
) map[string]swarmtypes.ServiceNetworkDetail {
if len(networkConfigs) == 0 {
@@ -135,7 +161,9 @@ func (s *Service) enrichServiceNetworkDetailsInternal(
details := make(map[string]swarmtypes.ServiceNetworkDetail, len(networkConfigs))
for _, networkConfig := range networkConfigs {
networkID := networkConfig.Target
- netInspectResult, err := dockerClient.NetworkInspect(ctx, networkID, client.NetworkInspectOptions{})
+ netInspectResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {
+ return dockerClient.NetworkInspect(ctx, networkID, client.NetworkInspectOptions{})
+ })
if err != nil {
continue
}
@@ -153,15 +181,17 @@ func (s *Service) enrichServiceNetworkDetailsInternal(
EnableIPv6: networkInfo.EnableIPv6,
ConfigOnly: networkInfo.ConfigOnly,
Options: networkInfo.Options,
+ ConfigFrom: networkInfo.ConfigFrom.Network,
}
- detail.ConfigFrom = cmp.Or(networkInfo.ConfigFrom.Network, detail.ConfigFrom)
for _, ipamCfg := range networkInfo.IPAM.Config {
- detail.IPAMConfigs = append(detail.IPAMConfigs, toServiceNetworkIPAMConfigInternal(ipamCfg))
+ detail.IPAMConfigs = append(detail.IPAMConfigs, toServiceNetworkIPAMConfig(ipamCfg))
}
if detail.ConfigFrom != "" {
- configInspectResult, networkInspectErr := dockerClient.NetworkInspect(ctx, detail.ConfigFrom, client.NetworkInspectOptions{})
+ configInspectResult, networkInspectErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkInspectResult, error) {
+ return dockerClient.NetworkInspect(ctx, detail.ConfigFrom, client.NetworkInspectOptions{})
+ })
if networkInspectErr == nil {
configNetwork := configInspectResult.Network
configDetail := &swarmtypes.ServiceNetworkConfigDetail{
@@ -173,7 +203,7 @@ func (s *Service) enrichServiceNetworkDetailsInternal(
Options: configNetwork.Options,
}
for _, ipamCfg := range configNetwork.IPAM.Config {
- converted := toServiceNetworkIPAMConfigInternal(ipamCfg)
+ converted := toServiceNetworkIPAMConfig(ipamCfg)
if ipamCfg.Subnet.IsValid() && ipamCfg.Subnet.Addr().Is6() {
configDetail.IPv6Configs = append(configDetail.IPv6Configs, converted)
continue
@@ -190,9 +220,8 @@ func (s *Service) enrichServiceNetworkDetailsInternal(
return details
}
-func (s *Service) enrichServiceMountsInternal(
+func (s *Service) enrichServiceMounts(
ctx context.Context,
- dockerClient *client.Client,
containerSpec *swarm.ContainerSpec,
) []swarmtypes.ServiceMount {
if containerSpec == nil {
@@ -201,22 +230,27 @@ func (s *Service) enrichServiceMountsInternal(
mounts := make([]swarmtypes.ServiceMount, 0, len(containerSpec.Mounts))
for _, serviceMount := range containerSpec.Mounts {
- mount := swarmtypes.ServiceMount{
- Type: string(serviceMount.Type),
- Source: serviceMount.Source,
- Target: serviceMount.Target,
- ReadOnly: serviceMount.ReadOnly,
- }
- if serviceMount.Type == "volume" && serviceMount.Source != "" {
- volInspectResult, err := dockerClient.VolumeInspect(ctx, serviceMount.Source, client.VolumeInspectOptions{})
- if err == nil {
- volume := volInspectResult.Volume
- mount.VolumeDriver = volume.Driver
- mount.VolumeOptions = volume.Options
- mount.DevicePath = volume.Mountpoint
+ volumeInfo := func() client.VolumeInspectResult {
+ if serviceMount.Type != "volume" || serviceMount.Source == "" {
+ return client.VolumeInspectResult{}
}
- }
- mounts = append(mounts, mount)
+ result, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, serviceMount.Source, client.VolumeInspectOptions{})
+ })
+ if err != nil {
+ return client.VolumeInspectResult{}
+ }
+ return result
+ }().Volume
+ mounts = append(mounts, swarmtypes.ServiceMount{
+ Type: string(serviceMount.Type),
+ Source: serviceMount.Source,
+ Target: serviceMount.Target,
+ ReadOnly: serviceMount.ReadOnly,
+ VolumeDriver: volumeInfo.Driver,
+ VolumeOptions: volumeInfo.Options,
+ DevicePath: volumeInfo.Mountpoint,
+ })
}
return mounts
@@ -227,11 +261,6 @@ func (s *Service) CreateService(ctx context.Context, req swarmtypes.ServiceCreat
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
// Unmarshal spec from JSON
var spec swarm.ServiceSpec
if unmarshalErr := json.Unmarshal(req.Spec, &spec); unmarshalErr != nil {
@@ -244,12 +273,14 @@ func (s *Service) CreateService(ctx context.Context, req swarmtypes.ServiceCreat
}
// Sanitize spec to avoid empty UID/GID in secret/config refs
- sanitizeServiceSpecInternal(&spec)
-
- resp, err := dockerClient.ServiceCreate(ctx, client.ServiceCreateOptions{
- Spec: spec,
- EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,
- QueryRegistry: optionsPayload.QueryRegistry,
+ sanitizeServiceSpec(&spec)
+
+ resp, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceCreateResult, error) {
+ return dockerClient.ServiceCreate(ctx, client.ServiceCreateOptions{
+ Spec: spec,
+ EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,
+ QueryRegistry: optionsPayload.QueryRegistry,
+ })
})
if err != nil {
return nil, fmt.Errorf("failed to create swarm service: %w", err)
@@ -266,14 +297,11 @@ func (s *Service) UpdateService(ctx context.Context, serviceID string, req swarm
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
versionIndex := req.Version
if versionIndex == 0 {
- serviceResult, serviceInspectErr := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ serviceResult, serviceInspectErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {
+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ })
if serviceInspectErr != nil {
return nil, fmt.Errorf("failed to inspect swarm service: %w", serviceInspectErr)
}
@@ -286,15 +314,17 @@ func (s *Service) UpdateService(ctx context.Context, serviceID string, req swarm
}
// Sanitize spec to avoid empty UID/GID in secret/config refs
- sanitizeServiceSpecInternal(&req.Spec)
-
- resp, err := dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{
- Version: swarm.Version{Index: versionIndex},
- Spec: req.Spec,
- EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,
- RegistryAuthFrom: optionsPayload.RegistryAuthFrom,
- Rollback: optionsPayload.Rollback,
- QueryRegistry: optionsPayload.QueryRegistry,
+ sanitizeServiceSpec(&req.Spec)
+
+ resp, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceUpdateResult, error) {
+ return dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{
+ Version: swarm.Version{Index: versionIndex},
+ Spec: req.Spec,
+ EncodedRegistryAuth: optionsPayload.EncodedRegistryAuth,
+ RegistryAuthFrom: optionsPayload.RegistryAuthFrom,
+ Rollback: optionsPayload.Rollback,
+ QueryRegistry: optionsPayload.QueryRegistry,
+ })
})
if err != nil {
return nil, fmt.Errorf("failed to update swarm service: %w", err)
@@ -310,12 +340,9 @@ func (s *Service) RemoveService(ctx context.Context, serviceID string) error {
return err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, serviceRemoveErr := dockerClient.ServiceRemove(ctx, serviceID, client.ServiceRemoveOptions{}); serviceRemoveErr != nil {
+ if _, serviceRemoveErr := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceRemoveResult, error) {
+ return dockerClient.ServiceRemove(ctx, serviceID, client.ServiceRemoveOptions{})
+ }); serviceRemoveErr != nil {
return fmt.Errorf("failed to remove swarm service: %w", serviceRemoveErr)
}
@@ -327,7 +354,7 @@ func (s *Service) StreamServiceLogs(ctx context.Context, serviceID string, logsC
return err
}
- dockerClient, err := s.dockerClient(ctx)
+ dockerClient, err := s.docker.GetClient(ctx)
if err != nil {
return fmt.Errorf("failed to connect to Docker: %w", err)
}
@@ -370,20 +397,19 @@ func (s *Service) RollbackService(ctx context.Context, serviceID string) (*swarm
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- serviceResult, err := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ serviceResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {
+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm service: %w", err)
}
- updateResult, err := dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{
- Version: serviceResult.Service.Version,
- Spec: serviceResult.Service.Spec,
- Rollback: "previous",
+ updateResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceUpdateResult, error) {
+ return dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{
+ Version: serviceResult.Service.Version,
+ Spec: serviceResult.Service.Spec,
+ Rollback: "previous",
+ })
})
if err != nil {
return nil, fmt.Errorf("failed to rollback swarm service: %w", err)
@@ -397,24 +423,28 @@ func (s *Service) ScaleService(ctx context.Context, serviceID string, replicas u
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- serviceResult, err := dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ serviceResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceInspectResult, error) {
+ return dockerClient.ServiceInspect(ctx, serviceID, client.ServiceInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm service: %w", err)
}
service := serviceResult.Service
- if applySwarmServiceScaleErr := applySwarmServiceScaleInternal(&service.Spec.Mode, replicas); applySwarmServiceScaleErr != nil {
- return nil, applySwarmServiceScaleErr
+ switch {
+ case service.Spec.Mode.Replicated != nil:
+ service.Spec.Mode.Replicated.Replicas = &replicas
+ case service.Spec.Mode.ReplicatedJob != nil:
+ service.Spec.Mode.ReplicatedJob.TotalCompletions = &replicas
+ default:
+ return nil, fmt.Errorf("scale can only be used with replicated or replicated-job mode: %w", errdefs.ErrInvalidArgument)
}
- updateResult, err := dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{
- Version: service.Version,
- Spec: service.Spec,
+ updateResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceUpdateResult, error) {
+ return dockerClient.ServiceUpdate(ctx, serviceID, client.ServiceUpdateOptions{
+ Version: service.Version,
+ Spec: service.Spec,
+ })
})
if err != nil {
return nil, fmt.Errorf("failed to scale swarm service: %w", err)
@@ -423,8 +453,10 @@ func (s *Service) ScaleService(ctx context.Context, serviceID string, replicas u
return &swarmtypes.ServiceUpdateResponse{Warnings: updateResult.Warnings}, nil
}
-func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *client.Client, services []swarm.Service) ([]swarmtypes.ServiceSummary, error) {
- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})
+func (s *Service) summarizeServices(ctx context.Context, services []swarm.Service) ([]swarmtypes.ServiceSummary, error) {
+ nodesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {
+ return dockerClient.NodeList(ctx, client.NodeListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list swarm nodes: %w", err)
}
@@ -434,7 +466,9 @@ func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *c
nodeNameByID[node.ID] = node.Description.Hostname
}
- networksResult, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})
+ networksResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NetworkListResult, error) {
+ return dockerClient.NetworkList(ctx, client.NetworkListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list networks: %w", err)
}
@@ -449,7 +483,9 @@ func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *c
serviceIDs[service.ID] = struct{}{}
}
- tasksResult, err := dockerClient.TaskList(ctx, client.TaskListOptions{})
+ tasksResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {
+ return dockerClient.TaskList(ctx, client.TaskListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list swarm tasks: %w", err)
}
@@ -486,88 +522,34 @@ func (s *Service) summarizeServicesInternal(ctx context.Context, dockerClient *c
return summaries, nil
}
-func (s *Service) buildServicePaginationConfigInternal() pagination.Config[swarmtypes.ServiceSummary] {
- return pagination.Config[swarmtypes.ServiceSummary]{
- SearchAccessors: []pagination.SearchAccessor[swarmtypes.ServiceSummary]{
- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Name, nil },
- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Image, nil },
- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.ID, nil },
- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.StackName, nil },
- func(svc swarmtypes.ServiceSummary) (string, error) { return svc.Mode, nil },
- func(svc swarmtypes.ServiceSummary) (string, error) {
- return strings.Join(svc.Networks, " "), nil
- },
- func(svc swarmtypes.ServiceSummary) (string, error) {
- return strings.Join(svc.Nodes, " "), nil
- },
- },
- SortBindings: []pagination.SortBinding[swarmtypes.ServiceSummary]{
- {Key: "name", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Name, b.Name) }},
- {Key: "image", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Image, b.Image) }},
- {Key: "mode", Fn: func(a, b swarmtypes.ServiceSummary) int { return strings.Compare(a.Mode, b.Mode) }},
- {Key: "replicas", Fn: func(a, b swarmtypes.ServiceSummary) int {
- if localCmp := cmp.Compare(a.Replicas, b.Replicas); localCmp != 0 {
- return localCmp
- }
- return cmp.Compare(a.RunningReplicas, b.RunningReplicas)
- }},
- {Key: "created", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},
- {Key: "updated", Fn: func(a, b swarmtypes.ServiceSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},
- },
- }
-}
-
-func sanitizeServiceSpecInternal(spec *swarm.ServiceSpec) {
+func sanitizeServiceSpec(spec *swarm.ServiceSpec) {
if spec == nil || spec.TaskTemplate.ContainerSpec == nil {
return
}
-
+ // Secret and config file targets default to root ownership.
+ defaultOwner := func(ids ...*string) {
+ for _, id := range ids {
+ if strings.TrimSpace(*id) == "" {
+ *id = "0"
+ }
+ }
+ }
for _, ref := range spec.TaskTemplate.ContainerSpec.Secrets {
if ref != nil && ref.File != nil {
- if strings.TrimSpace(ref.File.UID) == "" {
- ref.File.UID = "0"
- }
- if strings.TrimSpace(ref.File.GID) == "" {
- ref.File.GID = "0"
- }
+ defaultOwner(&ref.File.UID, &ref.File.GID)
}
}
-
for _, ref := range spec.TaskTemplate.ContainerSpec.Configs {
if ref != nil && ref.File != nil {
- if strings.TrimSpace(ref.File.UID) == "" {
- ref.File.UID = "0"
- }
- if strings.TrimSpace(ref.File.GID) == "" {
- ref.File.GID = "0"
- }
+ defaultOwner(&ref.File.UID, &ref.File.GID)
}
}
}
-func toServiceNetworkIPAMConfigInternal(cfg network.IPAMConfig) swarmtypes.ServiceNetworkIPAMConfig {
- out := swarmtypes.ServiceNetworkIPAMConfig{}
- if cfg.Subnet.IsValid() {
- out.Subnet = cfg.Subnet.String()
+func toServiceNetworkIPAMConfig(cfg network.IPAMConfig) swarmtypes.ServiceNetworkIPAMConfig {
+ return swarmtypes.ServiceNetworkIPAMConfig{
+ Subnet: kit.Ternary(cfg.Subnet.IsValid(), cfg.Subnet.String(), ""),
+ Gateway: kit.Ternary(cfg.Gateway.IsValid(), cfg.Gateway.String(), ""),
+ IPRange: kit.Ternary(cfg.IPRange.IsValid(), cfg.IPRange.String(), ""),
}
- if cfg.Gateway.IsValid() {
- out.Gateway = cfg.Gateway.String()
- }
- if cfg.IPRange.IsValid() {
- out.IPRange = cfg.IPRange.String()
- }
- return out
-}
-
-func applySwarmServiceScaleInternal(mode *swarm.ServiceMode, replicas uint64) error {
- switch {
- case mode.Replicated != nil:
- mode.Replicated.Replicas = &replicas
- case mode.ReplicatedJob != nil:
- mode.ReplicatedJob.TotalCompletions = &replicas
- default:
- return fmt.Errorf("scale can only be used with replicated or replicated-job mode: %w", errdefs.ErrInvalidArgument)
- }
-
- return nil
}
diff --git a/backend/internal/swarm/children/services/service_test.go b/backend/internal/swarm/children/services/service_test.go
index b9b7cbf74..32553e80e 100644
--- a/backend/internal/swarm/children/services/service_test.go
+++ b/backend/internal/swarm/children/services/service_test.go
@@ -17,7 +17,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
)
-func newSwarmTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
+func newSwarmTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
cli, err := client.New(
client.WithHost(server.URL),
@@ -29,7 +29,7 @@ func newSwarmTestDockerClientInternal(t *testing.T, server *httptest.Server) *cl
return cli
}
-func TestService_ScaleService_HandlesServiceModesInternal(t *testing.T) {
+func TestService_ScaleService_HandlesServiceModes(t *testing.T) {
ctx := t.Context()
replicas := uint64(5)
maxConcurrent := uint64(2)
@@ -122,8 +122,8 @@ func TestService_ScaleService_HandlesServiceModesInternal(t *testing.T) {
}))
t.Cleanup(server.Close)
- dockerService := &docker.DockerClientService{Client: newSwarmTestDockerClientInternal(t, server)}
- svc := NewService(dockerService.GetClient, func(context.Context) error { return nil }, nil)
+ dockerService := &docker.DockerClientService{Client: newSwarmTestDockerClient(t, server)}
+ svc := NewService(dockerService, func(context.Context) error { return nil }, nil)
resp, err := svc.ScaleService(ctx, "service-1", replicas)
if tt.wantErr {
diff --git a/backend/internal/swarm/children/stacks/service.go b/backend/internal/swarm/children/stacks/service.go
index b1b966deb..3dd1fdcb9 100644
--- a/backend/internal/swarm/children/stacks/service.go
+++ b/backend/internal/swarm/children/stacks/service.go
@@ -9,6 +9,7 @@ import (
"log/slog"
"path"
"path/filepath"
+ "slices"
"strings"
"time"
@@ -18,9 +19,9 @@ import (
"github.com/moby/moby/client"
"go.getarcane.app/acfs"
"go.getarcane.app/acfs/types"
- "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
libswarm "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/swarm"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
@@ -37,24 +38,24 @@ const (
// Service deploys and manages swarm stacks and their persisted sources.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
+ docker *dockerInternal.DockerClientService
ensureManager func(ctx context.Context) error
listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error)
- paginateServices func(ctx context.Context, dockerClient *client.Client, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error)
+ paginateServices func(ctx context.Context, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error)
registryAuth func(ctx context.Context, imageRef string) (string, error)
settingsService *settings.SettingsService
}
func NewService(
- dockerClient func(ctx context.Context) (*client.Client, error),
+ dockerService *dockerInternal.DockerClientService,
ensureManager func(ctx context.Context) error,
listTasks func(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error),
- paginateServices func(ctx context.Context, dockerClient *client.Client, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error),
+ paginateServices func(ctx context.Context, services []swarm.Service, params pagination.QueryParams) ([]swarmtypes.ServiceSummary, pagination.Response, error),
registryAuth func(ctx context.Context, imageRef string) (string, error),
settingsService *settings.SettingsService,
) *Service {
return &Service{
- dockerClient: dockerClient,
+ docker: dockerService,
ensureManager: ensureManager,
listTasks: listTasks,
paginateServices: paginateServices,
@@ -68,12 +69,9 @@ func (s *Service) ListStacksPaginated(ctx context.Context, environmentID string,
return nil, pagination.Response{}, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{})
+ servicesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {
+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{})
+ })
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list swarm services: %w", err)
}
@@ -108,7 +106,7 @@ func (s *Service) ListStacksPaginated(ctx context.Context, environmentID string,
}
}
- persistedStacks, err := s.listPersistedStackSourcesInternal(ctx, environmentID)
+ persistedStacks, err := s.listPersistedStackSources(ctx, environmentID)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -124,7 +122,18 @@ func (s *Service) ListStacksPaginated(ctx context.Context, environmentID string,
items = append(items, *stack)
}
- config := s.buildStackPaginationConfigInternal()
+ config := pagination.Config[swarmtypes.StackSummary]{
+ SearchAccessors: []pagination.SearchAccessor[swarmtypes.StackSummary]{
+ func(stack swarmtypes.StackSummary) (string, error) { return stack.Name, nil },
+ func(stack swarmtypes.StackSummary) (string, error) { return stack.Namespace, nil },
+ },
+ SortBindings: []pagination.SortBinding[swarmtypes.StackSummary]{
+ {Key: "name", Fn: func(a, b swarmtypes.StackSummary) int { return strings.Compare(a.Name, b.Name) }},
+ {Key: "services", Fn: func(a, b swarmtypes.StackSummary) int { return cmp.Compare(a.Services, b.Services) }},
+ {Key: "created", Fn: func(a, b swarmtypes.StackSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},
+ {Key: "updated", Fn: func(a, b swarmtypes.StackSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},
+ },
+ }
result := config.SearchOrderAndPaginate(items, params)
paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params)
@@ -141,18 +150,18 @@ func (s *Service) DeployStack(ctx context.Context, environmentID string, req swa
return nil, errors.New("stack name is required")
}
- dockerClient, err := s.dockerClient(ctx)
+ dockerClient, err := s.docker.GetClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)
+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)
if err != nil {
return nil, err
}
- if upsertStackSourceErr := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {
- slog.WarnContext(ctx, "failed to persist swarm stack source", "environmentId", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", upsertStackSourceErr)
+ if upsertStackSourceErr := s.upsertStackSource(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {
+ slog.WarnContext(ctx, "failed to persist swarm stack source", "environmentId", normalizeSwarmEnvironmentID(environmentID), "stackName", stackName, "error", upsertStackSourceErr)
}
workingDir := req.WorkingDir
@@ -184,7 +193,7 @@ func (s *Service) DeployStack(ctx context.Context, environmentID string, req swa
WorkingDir: workingDir,
PathMapper: pm,
}); deployStackErr != nil {
- return nil, classifySwarmStackErrorInternal(deployStackErr)
+ return nil, classifySwarmStackError(deployStackErr)
}
return &swarmtypes.StackDeployResponse{Name: stackName}, nil
@@ -195,19 +204,14 @@ func (s *Service) GetStack(ctx context.Context, environmentID, stackName string)
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)
+ services, err := s.listStackServicesRaw(ctx, stackName)
if err != nil {
return nil, err
}
if len(services) == 0 {
- persisted, getPersistedStackSourceSummaryErr := s.getPersistedStackSourceSummaryInternal(ctx, environmentID, stackName)
- if getPersistedStackSourceSummaryErr != nil {
- return nil, kit.Ternary[error](errdefs.IsNotFound(getPersistedStackSourceSummaryErr), errdefs.ErrNotFound, getPersistedStackSourceSummaryErr)
+ persisted, persistedErr := s.getPersistedStackSourceSummary(ctx, environmentID, stackName)
+ if persistedErr != nil {
+ return nil, persistedErr
}
return &swarmtypes.StackInspect{
@@ -245,7 +249,7 @@ func (s *Service) GetStackSource(ctx context.Context, environmentID, stackName s
return nil, errors.New("stack name is required")
}
- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)
+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)
if err != nil {
return nil, err
}
@@ -306,7 +310,7 @@ func (s *Service) GetStackSource(ctx context.Context, environmentID, stackName s
}, nil
}
-var deployStackAfterSourceUpdateInternal = (*Service).DeployStack
+var deployStackAfterSourceUpdate = (*Service).DeployStack
func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackName string, req swarmtypes.StackSourceUpdateRequest) (*swarmtypes.StackSource, error) {
stackName = strings.TrimSpace(stackName)
@@ -322,7 +326,7 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam
return nil, err
}
- if upsertStackSourceErr := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {
+ if upsertStackSourceErr := s.upsertStackSource(ctx, environmentID, stackName, req.ComposeContent, req.OverrideContent, req.EnvContent, req.Files); upsertStackSourceErr != nil {
return nil, upsertStackSourceErr
}
@@ -334,7 +338,7 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam
// resolution is per image and yields nothing unless a configured container
// registry matches the image host. Without it, a service added in the edit
// has no previous spec to fall back on and private images fail to pull.
- if _, deployStackAfterSourceUpdateErr := deployStackAfterSourceUpdateInternal(s, ctx, environmentID, swarmtypes.StackDeployRequest{
+ if _, deployStackAfterSourceUpdateErr := deployStackAfterSourceUpdate(s, ctx, environmentID, swarmtypes.StackDeployRequest{
Name: stackName,
ComposeContent: req.ComposeContent,
OverrideContent: req.OverrideContent,
@@ -347,12 +351,12 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam
// was never successfully deployed.
var restoreErr error
if previous != nil {
- restoreErr = s.upsertStackSourceInternal(ctx, environmentID, stackName, previous.ComposeContent, previous.OverrideContent, previous.EnvContent, previous.Files)
+ restoreErr = s.upsertStackSource(ctx, environmentID, stackName, previous.ComposeContent, previous.OverrideContent, previous.EnvContent, previous.Files)
} else {
- restoreErr = s.deleteStackSourceInternal(ctx, environmentID, stackName)
+ restoreErr = s.deleteStackSource(ctx, environmentID, stackName)
}
if restoreErr != nil {
- slog.WarnContext(ctx, "failed to restore swarm stack source after deploy failure", "environmentId", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", restoreErr)
+ slog.WarnContext(ctx, "failed to restore swarm stack source after deploy failure", "environmentId", normalizeSwarmEnvironmentID(environmentID), "stackName", stackName, "error", restoreErr)
}
return nil, fmt.Errorf("failed to redeploy swarm stack from updated source: %w", deployStackAfterSourceUpdateErr)
}
@@ -366,8 +370,8 @@ func (s *Service) UpdateStackSource(ctx context.Context, environmentID, stackNam
}, nil
}
-func (s *Service) listPersistedStackSourcesInternal(ctx context.Context, environmentID string) (map[string]swarmtypes.StackSummary, error) {
- _, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDirInternal(ctx, environmentID)
+func (s *Service) listPersistedStackSources(ctx context.Context, environmentID string) (map[string]swarmtypes.StackSummary, error) {
+ _, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDir(ctx, environmentID)
if err != nil {
return nil, err
}
@@ -386,7 +390,7 @@ func (s *Service) listPersistedStackSourcesInternal(ctx context.Context, environ
continue
}
- summary, buildPersistedStackSourceSummaryErr := s.buildPersistedStackSourceSummaryInternal(ctx, filepath.Join(environmentDir, entry.Name), entry.Name)
+ summary, buildPersistedStackSourceSummaryErr := s.buildPersistedStackSourceSummary(ctx, filepath.Join(environmentDir, entry.Name), entry.Name)
if buildPersistedStackSourceSummaryErr != nil {
if errdefs.IsNotFound(buildPersistedStackSourceSummaryErr) {
continue
@@ -400,16 +404,21 @@ func (s *Service) listPersistedStackSourcesInternal(ctx context.Context, environ
return stacks, nil
}
-func (s *Service) getPersistedStackSourceSummaryInternal(ctx context.Context, environmentID, stackName string) (*swarmtypes.StackSummary, error) {
- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)
+// getPersistedStackSourceSummary maps any not-found failure to errdefs.ErrNotFound.
+func (s *Service) getPersistedStackSourceSummary(ctx context.Context, environmentID, stackName string) (*swarmtypes.StackSummary, error) {
+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)
if err != nil {
return nil, err
}
- return s.buildPersistedStackSourceSummaryInternal(ctx, stackSourceDir, stackName)
+ summary, err := s.buildPersistedStackSourceSummary(ctx, stackSourceDir, stackName)
+ if errdefs.IsNotFound(err) {
+ return nil, errdefs.ErrNotFound
+ }
+ return summary, err
}
-func (s *Service) buildPersistedStackSourceSummaryInternal(ctx context.Context, stackSourceDir, stackName string) (*swarmtypes.StackSummary, error) {
+func (s *Service) buildPersistedStackSourceSummary(ctx context.Context, stackSourceDir, stackName string) (*swarmtypes.StackSummary, error) {
composeEntry, err := acfs.Stat(ctx, stackSourceDir, "/"+swarmStackComposeFilename, true)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
@@ -453,46 +462,48 @@ func (s *Service) RemoveStack(ctx context.Context, environmentID, stackName stri
return errors.New("stack name is required")
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)
+ services, err := s.listStackServicesRaw(ctx, stackName)
if err != nil {
return err
}
if len(services) == 0 {
- if _, getPersistedStackSourceSummaryErr := s.getPersistedStackSourceSummaryInternal(ctx, environmentID, stackName); getPersistedStackSourceSummaryErr != nil {
- return kit.Ternary[error](errdefs.IsNotFound(getPersistedStackSourceSummaryErr), errdefs.ErrNotFound, getPersistedStackSourceSummaryErr)
+ if _, persistedErr := s.getPersistedStackSourceSummary(ctx, environmentID, stackName); persistedErr != nil {
+ return persistedErr
}
} else {
- if removeStackServicesErr := s.removeStackServicesInternal(ctx, dockerClient, services); removeStackServicesErr != nil {
+ if removeStackServicesErr := s.removeStackServices(ctx, services); removeStackServicesErr != nil {
return removeStackServicesErr
}
}
+ dockerClient, err := s.docker.GetClient(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to connect to Docker: %w", err)
+ }
+
if removeStackResourcesErr := libswarm.RemoveStackResources(ctx, dockerClient, stackName); removeStackResourcesErr != nil {
- return classifySwarmStackErrorInternal(removeStackResourcesErr)
+ return classifySwarmStackError(removeStackResourcesErr)
}
- if deleteStackSourceErr := s.deleteStackSourceInternal(ctx, environmentID, stackName); deleteStackSourceErr != nil {
- slog.WarnContext(ctx, "failed to remove persisted swarm stack source", "environmentId", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", deleteStackSourceErr)
+ if deleteStackSourceErr := s.deleteStackSource(ctx, environmentID, stackName); deleteStackSourceErr != nil {
+ slog.WarnContext(ctx, "failed to remove persisted swarm stack source", "environmentId", normalizeSwarmEnvironmentID(environmentID), "stackName", stackName, "error", deleteStackSourceErr)
}
return nil
}
-func (s *Service) removeStackServicesInternal(ctx context.Context, dockerClient *client.Client, services []swarm.Service) error {
+func (s *Service) removeStackServices(ctx context.Context, services []swarm.Service) error {
serviceIDs := make(map[string]struct{}, len(services))
for _, service := range services {
serviceIDs[service.ID] = struct{}{}
- if _, err := dockerClient.ServiceRemove(ctx, service.ID, client.ServiceRemoveOptions{}); err != nil && !errdefs.IsNotFound(err) {
+ if _, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceRemoveResult, error) {
+ return dockerClient.ServiceRemove(ctx, service.ID, client.ServiceRemoveOptions{})
+ }); err != nil && !errdefs.IsNotFound(err) {
return fmt.Errorf("failed to remove swarm service %s: %w", service.Spec.Name, err)
}
}
- if err := s.waitForRemovedServiceTasksInternal(ctx, dockerClient, serviceIDs, 30*time.Second); err != nil {
+ if err := s.waitForRemovedServiceTasks(ctx, serviceIDs, 30*time.Second); err != nil {
return err
}
@@ -504,12 +515,7 @@ func (s *Service) ListStackServicesPaginated(ctx context.Context, stackName stri
return nil, pagination.Response{}, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)
+ services, err := s.listStackServicesRaw(ctx, stackName)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -517,7 +523,7 @@ func (s *Service) ListStackServicesPaginated(ctx context.Context, stackName stri
return nil, pagination.Response{}, errdefs.ErrNotFound
}
- return s.paginateServices(ctx, dockerClient, services, params)
+ return s.paginateServices(ctx, services, params)
}
func (s *Service) ListStackTasksPaginated(ctx context.Context, stackName string, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {
@@ -525,12 +531,7 @@ func (s *Service) ListStackTasksPaginated(ctx context.Context, stackName string,
return nil, pagination.Response{}, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- services, err := s.listStackServicesRawInternal(ctx, dockerClient, stackName)
+ services, err := s.listStackServicesRaw(ctx, stackName)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -551,11 +552,11 @@ func (s *Service) RenderStackConfig(ctx context.Context, environmentID string, r
return nil, err
}
- _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, req.Name)
+ _, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, req.Name)
if err != nil {
return nil, err
}
- dockerClient, err := s.dockerClient(ctx)
+ dockerClient, err := s.docker.GetClient(ctx)
if err != nil {
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
@@ -575,7 +576,7 @@ func (s *Service) RenderStackConfig(ctx context.Context, environmentID string, r
PathMapper: pm,
})
if err != nil {
- return nil, classifySwarmStackErrorInternal(err)
+ return nil, classifySwarmStackError(err)
}
return &swarmtypes.StackRenderConfigResponse{
@@ -589,16 +590,18 @@ func (s *Service) RenderStackConfig(ctx context.Context, environmentID string, r
}, nil
}
-func (s *Service) listStackServicesRawInternal(ctx context.Context, dockerClient *client.Client, stackName string) ([]swarm.Service, error) {
+func (s *Service) listStackServicesRaw(ctx context.Context, stackName string) ([]swarm.Service, error) {
stackName = strings.TrimSpace(stackName)
if stackName == "" {
return nil, errors.New("stack name is required")
}
stackFilter := make(client.Filters).Add("label", fmt.Sprintf("%s=%s", swarmtypes.StackNamespaceLabel, stackName))
- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{
- Filters: stackFilter,
- Status: true,
+ servicesResult, err := s.docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {
+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{
+ Filters: stackFilter,
+ Status: true,
+ })
})
if err != nil {
return nil, fmt.Errorf("failed to list stack services: %w", err)
@@ -607,7 +610,7 @@ func (s *Service) listStackServicesRawInternal(ctx context.Context, dockerClient
return servicesResult.Items, nil
}
-func (s *Service) waitForRemovedServiceTasksInternal(ctx context.Context, dockerClient *client.Client, serviceIDs map[string]struct{}, timeout time.Duration) error {
+func (s *Service) waitForRemovedServiceTasks(ctx context.Context, serviceIDs map[string]struct{}, timeout time.Duration) error {
if len(serviceIDs) == 0 {
return nil
}
@@ -624,18 +627,23 @@ func (s *Service) waitForRemovedServiceTasksInternal(ctx context.Context, docker
}
for {
- tasksResult, err := dockerClient.TaskList(waitCtx, client.TaskListOptions{Filters: taskFilters})
+ tasksResult, err := s.docker.Call(waitCtx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {
+ return dockerClient.TaskList(ctx, client.TaskListOptions{Filters: taskFilters})
+ })
if err != nil {
return fmt.Errorf("failed to list tasks while waiting for stack removal: %w", err)
}
- hasActiveTasks := false
- for _, task := range tasksResult.Items {
- if !isTaskTerminalInternal(task.Status.State) {
- hasActiveTasks = true
- break
- }
- }
+ hasActiveTasks := slices.ContainsFunc(tasksResult.Items, func(task swarm.Task) bool {
+ return !slices.Contains([]swarm.TaskState{
+ swarm.TaskStateComplete,
+ swarm.TaskStateShutdown,
+ swarm.TaskStateFailed,
+ swarm.TaskStateRejected,
+ swarm.TaskStateRemove,
+ swarm.TaskStateOrphaned,
+ }, task.Status.State)
+ })
if !hasActiveTasks {
return nil
}
@@ -648,37 +656,13 @@ func (s *Service) waitForRemovedServiceTasksInternal(ctx context.Context, docker
}
}
-func isTaskTerminalInternal(state swarm.TaskState) bool {
- switch state {
- case swarm.TaskStateComplete,
- swarm.TaskStateShutdown,
- swarm.TaskStateFailed,
- swarm.TaskStateRejected,
- swarm.TaskStateRemove,
- swarm.TaskStateOrphaned:
- return true
- case swarm.TaskStateNew,
- swarm.TaskStateAllocated,
- swarm.TaskStatePending,
- swarm.TaskStateAssigned,
- swarm.TaskStateAccepted,
- swarm.TaskStatePreparing,
- swarm.TaskStateReady,
- swarm.TaskStateStarting,
- swarm.TaskStateRunning:
- return false
- }
-
- return false
-}
-
-func (s *Service) upsertStackSourceInternal(ctx context.Context, environmentID, stackName, composeContent, overrideContent, envContent string, files []swarmtypes.SyncFile) error {
+func (s *Service) upsertStackSource(ctx context.Context, environmentID, stackName, composeContent, overrideContent, envContent string, files []swarmtypes.SyncFile) error {
stackName = strings.TrimSpace(stackName)
if stackName == "" {
return errors.New("stack name is required")
}
- rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)
+ rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)
if err != nil {
return err
}
@@ -742,12 +726,12 @@ func (s *Service) upsertStackSourceInternal(ctx context.Context, environmentID,
return nil
}
-func (s *Service) deleteStackSourceInternal(ctx context.Context, environmentID, stackName string) error {
+func (s *Service) deleteStackSource(ctx context.Context, environmentID, stackName string) error {
if strings.TrimSpace(stackName) == "" {
return errors.New("stack name is required")
}
- rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)
+ rootDir, stackSourceDir, err := s.resolveSwarmStackSourceDir(ctx, environmentID, stackName)
if err != nil {
return err
}
@@ -771,18 +755,17 @@ func (s *Service) deleteStackSourceInternal(ctx context.Context, environmentID,
return nil
}
-func normalizeSwarmEnvironmentIDInternal(environmentID string) string {
- envID := strings.TrimSpace(environmentID)
- return kit.Ternary(envID == "", "0", envID)
+func normalizeSwarmEnvironmentID(environmentID string) string {
+ return cmp.Or(strings.TrimSpace(environmentID), "0")
}
-func (s *Service) resolveSwarmStackSourceDirInternal(ctx context.Context, environmentID, stackName string) (string, string, error) {
+func (s *Service) resolveSwarmStackSourceDir(ctx context.Context, environmentID, stackName string) (string, string, error) {
normalizedStackName := projects.SanitizeProjectName(strings.TrimSpace(stackName))
if normalizedStackName == "" || strings.Trim(normalizedStackName, "_") == "" {
return "", "", errors.New("invalid stack name")
}
- rootDir, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDirInternal(ctx, environmentID)
+ rootDir, environmentDir, err := s.resolveSwarmStackSourceEnvironmentDir(ctx, environmentID)
if err != nil {
return "", "", err
}
@@ -795,8 +778,8 @@ func (s *Service) resolveSwarmStackSourceDirInternal(ctx context.Context, enviro
return rootDir, stackSourceDir, nil
}
-func (s *Service) resolveSwarmStackSourceEnvironmentDirInternal(ctx context.Context, environmentID string) (string, string, error) {
- normalizedEnvironmentID := projects.SanitizeProjectName(normalizeSwarmEnvironmentIDInternal(environmentID))
+func (s *Service) resolveSwarmStackSourceEnvironmentDir(ctx context.Context, environmentID string) (string, string, error) {
+ normalizedEnvironmentID := projects.SanitizeProjectName(normalizeSwarmEnvironmentID(environmentID))
if normalizedEnvironmentID == "" || strings.Trim(normalizedEnvironmentID, "_") == "" {
normalizedEnvironmentID = "0"
}
@@ -815,22 +798,7 @@ func (s *Service) resolveSwarmStackSourceEnvironmentDirInternal(ctx context.Cont
return rootDir, environmentDir, nil
}
-func (s *Service) buildStackPaginationConfigInternal() pagination.Config[swarmtypes.StackSummary] {
- return pagination.Config[swarmtypes.StackSummary]{
- SearchAccessors: []pagination.SearchAccessor[swarmtypes.StackSummary]{
- func(stack swarmtypes.StackSummary) (string, error) { return stack.Name, nil },
- func(stack swarmtypes.StackSummary) (string, error) { return stack.Namespace, nil },
- },
- SortBindings: []pagination.SortBinding[swarmtypes.StackSummary]{
- {Key: "name", Fn: func(a, b swarmtypes.StackSummary) int { return strings.Compare(a.Name, b.Name) }},
- {Key: "services", Fn: func(a, b swarmtypes.StackSummary) int { return cmp.Compare(a.Services, b.Services) }},
- {Key: "created", Fn: func(a, b swarmtypes.StackSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},
- {Key: "updated", Fn: func(a, b swarmtypes.StackSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},
- },
- }
-}
-
-func classifySwarmStackErrorInternal(err error) error {
+func classifySwarmStackError(err error) error {
if errors.Is(err, libswarm.ErrInvalidStack) {
return common.Classify(common.ErrBadRequest, err)
}
diff --git a/backend/internal/swarm/children/stacks/service_test.go b/backend/internal/swarm/children/stacks/service_test.go
index 3560cc1bf..f3832924f 100644
--- a/backend/internal/swarm/children/stacks/service_test.go
+++ b/backend/internal/swarm/children/stacks/service_test.go
@@ -16,7 +16,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
)
-func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {
+func setupSwarmServiceTestDB(t *testing.T) *database.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -24,7 +24,7 @@ func setupSwarmServiceTestDBInternal(t *testing.T) *database.DB {
return &database.DB{DB: db}
}
-func newSettingsServiceForSwarmTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
+func newSettingsServiceForSwarmTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
t.Helper()
svc, err := settings.NewSettingsService(ctx, db)
if err == nil {
@@ -38,12 +38,12 @@ type stackSourceDeployRecorder struct {
lastReq swarm.StackDeployRequest
}
-func stubStackSourceUpdateDeployInternal(t *testing.T) *stackSourceDeployRecorder {
+func stubStackSourceUpdateDeploy(t *testing.T) *stackSourceDeployRecorder {
t.Helper()
- original := deployStackAfterSourceUpdateInternal
- t.Cleanup(func() { deployStackAfterSourceUpdateInternal = original })
+ original := deployStackAfterSourceUpdate
+ t.Cleanup(func() { deployStackAfterSourceUpdate = original })
rec := &stackSourceDeployRecorder{}
- deployStackAfterSourceUpdateInternal = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {
+ deployStackAfterSourceUpdate = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {
rec.calls++
rec.lastReq = req
return &swarm.StackDeployResponse{Name: req.Name}, nil
@@ -53,15 +53,15 @@ func stubStackSourceUpdateDeployInternal(t *testing.T) *stackSourceDeployRecorde
func TestService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManager(t *testing.T) {
ctx := t.Context()
- db := setupSwarmServiceTestDBInternal(t)
+ db := setupSwarmServiceTestDB(t)
rootDir := t.TempDir()
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)
- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)
+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)
require.NoError(t, err)
svc := NewService(nil, nil, nil, nil, nil, settingsSvc)
- deployRec := stubStackSourceUpdateDeployInternal(t)
+ deployRec := stubStackSourceUpdateDeploy(t)
updated, err := svc.UpdateStackSource(ctx, "0", "demo-stack", swarm.StackSourceUpdateRequest{
ComposeContent: "services:\n web:\n image: nginx:alpine\n",
@@ -117,15 +117,15 @@ func TestService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManager(t *t
func TestService_UpdateAndGetStackSource_RoundTripsOverride(t *testing.T) {
ctx := t.Context()
- db := setupSwarmServiceTestDBInternal(t)
+ db := setupSwarmServiceTestDB(t)
rootDir := t.TempDir()
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)
- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)
+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)
require.NoError(t, err)
svc := NewService(nil, nil, nil, nil, nil, settingsSvc)
- stubStackSourceUpdateDeployInternal(t)
+ stubStackSourceUpdateDeploy(t)
overridePath := filepath.Join(rootDir, "0", "demo-stack", "compose.override.yaml")
@@ -157,20 +157,20 @@ func TestService_UpdateAndGetStackSource_RoundTripsOverride(t *testing.T) {
func TestService_UpdateStackSource_PrunesAndRestoresOnDeployFailure(t *testing.T) {
ctx := t.Context()
- db := setupSwarmServiceTestDBInternal(t)
+ db := setupSwarmServiceTestDB(t)
rootDir := t.TempDir()
t.Setenv("SWARM_STACK_SOURCES_DIRECTORY", rootDir)
- settingsSvc, err := newSettingsServiceForSwarmTestInternal(t, ctx, db)
+ settingsSvc, err := newSettingsServiceForSwarmTest(t, ctx, db)
require.NoError(t, err)
svc := NewService(nil, nil, nil, nil, nil, settingsSvc)
- original := deployStackAfterSourceUpdateInternal
- t.Cleanup(func() { deployStackAfterSourceUpdateInternal = original })
+ original := deployStackAfterSourceUpdate
+ t.Cleanup(func() { deployStackAfterSourceUpdate = original })
var lastReq swarm.StackDeployRequest
var deployErr error
- deployStackAfterSourceUpdateInternal = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {
+ deployStackAfterSourceUpdate = func(_ *Service, _ context.Context, _ string, req swarm.StackDeployRequest) (*swarm.StackDeployResponse, error) {
lastReq = req
if deployErr != nil {
return nil, deployErr
diff --git a/backend/internal/swarm/service.go b/backend/internal/swarm/service.go
index eda963a8c..6c3b5346f 100644
--- a/backend/internal/swarm/service.go
+++ b/backend/internal/swarm/service.go
@@ -2,6 +2,7 @@ package swarm
import (
"bytes"
+ "cmp"
"context"
stdjson "encoding/json"
"encoding/json/v2"
@@ -15,7 +16,6 @@ import (
"github.com/moby/moby/api/types/swarm"
"github.com/moby/moby/api/types/system"
"github.com/moby/moby/client"
- "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
@@ -63,11 +63,11 @@ func NewSwarmService(
registryAuth = registryService.GetRegistryAuthForImage
}
- s.services = services.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal, s.listTasksPaginatedWithFiltersInternal)
- s.nodes = nodes.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal, s.listTasksPaginatedWithFiltersInternal, s.GetSwarmJoinTokens, environmentService)
- s.stacks = stacks.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal, s.listTasksPaginatedWithFiltersInternal, s.services.PaginateSummaries, registryAuth, settingsService)
- s.configs = configs.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal)
- s.secrets = secrets.NewService(dockerService.GetClient, s.ensureSwarmManagerInternal)
+ s.services = services.NewService(dockerService, s.ensureSwarmManager, s.listTasksPaginatedWithFilters)
+ s.nodes = nodes.NewService(dockerService, s.ensureSwarmManager, s.listTasksPaginatedWithFilters, s.GetSwarmJoinTokens, environmentService)
+ s.stacks = stacks.NewService(dockerService, s.ensureSwarmManager, s.listTasksPaginatedWithFilters, s.services.PaginateSummaries, registryAuth, settingsService)
+ s.configs = configs.NewService(dockerService, s.ensureSwarmManager)
+ s.secrets = secrets.NewService(dockerService, s.ensureSwarmManager)
return s
}
@@ -95,24 +95,21 @@ func (s *SwarmService) IsEnabled(ctx context.Context) (bool, error) {
}
func (s *SwarmService) ListTasksPaginated(ctx context.Context, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {
- if err := s.ensureSwarmManagerInternal(ctx); err != nil {
+ if err := s.ensureSwarmManager(ctx); err != nil {
return nil, pagination.Response{}, err
}
- return s.listTasksPaginatedWithFiltersInternal(ctx, nil, params)
+ return s.listTasksPaginatedWithFilters(ctx, nil, params)
}
func (s *SwarmService) GetSwarmInfo(ctx context.Context) (*swarmtypes.SwarmInfo, error) {
- if err := s.ensureSwarmManagerInternal(ctx); err != nil {
+ if err := s.ensureSwarmManager(ctx); err != nil {
return nil, err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- infoResult, err := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {
+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm: %w", err)
}
@@ -121,12 +118,7 @@ func (s *SwarmService) GetSwarmInfo(ctx context.Context) (*swarmtypes.SwarmInfo,
}
func (s *SwarmService) InitSwarm(ctx context.Context, req swarmtypes.SwarmInitRequest) (*swarmtypes.SwarmInitResponse, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- spec, err := decodeSwarmSpecInternal(req.Spec)
+ spec, err := decodeSwarmSpec(req.Spec)
if err != nil {
return nil, err
}
@@ -140,79 +132,72 @@ func (s *SwarmService) InitSwarm(ctx context.Context, req swarmtypes.SwarmInitRe
defaultAddrPool = append(defaultAddrPool, prefix.Masked())
}
- initResult, err := dockerClient.SwarmInit(ctx, client.SwarmInitOptions{
- ListenAddr: defaultSwarmListenAddrInternal(req.ListenAddr),
- AdvertiseAddr: req.AdvertiseAddr,
- DataPathAddr: req.DataPathAddr,
- DataPathPort: req.DataPathPort,
- ForceNewCluster: req.ForceNewCluster,
- Spec: spec,
- AutoLockManagers: req.AutoLockManagers,
- Availability: req.Availability,
- DefaultAddrPool: defaultAddrPool,
- SubnetSize: req.SubnetSize,
+ initResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInitResult, error) {
+ return dockerClient.SwarmInit(ctx, client.SwarmInitOptions{
+ ListenAddr: cmp.Or(strings.TrimSpace(req.ListenAddr), defaultSwarmListenAddr),
+ AdvertiseAddr: req.AdvertiseAddr,
+ DataPathAddr: req.DataPathAddr,
+ DataPathPort: req.DataPathPort,
+ ForceNewCluster: req.ForceNewCluster,
+ Spec: spec,
+ AutoLockManagers: req.AutoLockManagers,
+ Availability: req.Availability,
+ DefaultAddrPool: defaultAddrPool,
+ SubnetSize: req.SubnetSize,
+ })
})
if err != nil {
return nil, fmt.Errorf("failed to initialize swarm: %w", err)
}
- s.persistSwarmEnabledStateInternal(ctx, true)
+ s.persistSwarmEnabledState(ctx, true)
return &swarmtypes.SwarmInitResponse{NodeID: initResult.NodeID}, nil
}
func (s *SwarmService) JoinSwarm(ctx context.Context, req swarmtypes.SwarmJoinRequest) error {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, swarmJoinErr := dockerClient.SwarmJoin(ctx, client.SwarmJoinOptions{
- ListenAddr: defaultSwarmListenAddrInternal(req.ListenAddr),
- AdvertiseAddr: req.AdvertiseAddr,
- DataPathAddr: req.DataPathAddr,
- RemoteAddrs: req.RemoteAddrs,
- JoinToken: req.JoinToken,
- Availability: req.Availability,
+ if _, swarmJoinErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmJoinResult, error) {
+ return dockerClient.SwarmJoin(ctx, client.SwarmJoinOptions{
+ ListenAddr: cmp.Or(strings.TrimSpace(req.ListenAddr), defaultSwarmListenAddr),
+ AdvertiseAddr: req.AdvertiseAddr,
+ DataPathAddr: req.DataPathAddr,
+ RemoteAddrs: req.RemoteAddrs,
+ JoinToken: req.JoinToken,
+ Availability: req.Availability,
+ })
}); swarmJoinErr != nil {
return fmt.Errorf("failed to join swarm: %w", swarmJoinErr)
}
- s.persistSwarmEnabledStateInternal(ctx, true)
+ s.persistSwarmEnabledState(ctx, true)
return nil
}
func (s *SwarmService) LeaveSwarm(ctx context.Context, req swarmtypes.SwarmLeaveRequest) error {
- if err := s.ensureSwarmActiveInternal(ctx); err != nil {
+ if _, err := s.ensureSwarmActive(ctx); err != nil {
return err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, swarmLeaveErr := dockerClient.SwarmLeave(ctx, client.SwarmLeaveOptions{Force: req.Force}); swarmLeaveErr != nil {
+ if _, swarmLeaveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmLeaveResult, error) {
+ return dockerClient.SwarmLeave(ctx, client.SwarmLeaveOptions{Force: req.Force})
+ }); swarmLeaveErr != nil {
return fmt.Errorf("failed to leave swarm: %w", swarmLeaveErr)
}
- s.persistSwarmEnabledStateInternal(ctx, false)
+ s.persistSwarmEnabledState(ctx, false)
return nil
}
func (s *SwarmService) UnlockSwarm(ctx context.Context, req swarmtypes.SwarmUnlockRequest) error {
- if err := s.ensureSwarmActiveInternal(ctx); err != nil {
+ if _, err := s.ensureSwarmActive(ctx); err != nil {
return err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- if _, swarmUnlockErr := dockerClient.SwarmUnlock(ctx, client.SwarmUnlockOptions{Key: req.Key}); swarmUnlockErr != nil {
+ if _, swarmUnlockErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmUnlockResult, error) {
+ return dockerClient.SwarmUnlock(ctx, client.SwarmUnlockOptions{Key: req.Key})
+ }); swarmUnlockErr != nil {
return fmt.Errorf("failed to unlock swarm: %w", swarmUnlockErr)
}
@@ -220,16 +205,13 @@ func (s *SwarmService) UnlockSwarm(ctx context.Context, req swarmtypes.SwarmUnlo
}
func (s *SwarmService) GetSwarmUnlockKey(ctx context.Context) (*swarmtypes.SwarmUnlockKeyResponse, error) {
- if err := s.ensureSwarmManagerInternal(ctx); err != nil {
+ if err := s.ensureSwarmManager(ctx); err != nil {
return nil, err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- unlockResult, err := dockerClient.SwarmGetUnlockKey(ctx)
+ unlockResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmGetUnlockKeyResult, error) {
+ return dockerClient.SwarmGetUnlockKey(ctx)
+ })
if err != nil {
return nil, fmt.Errorf("failed to get swarm unlock key: %w", err)
}
@@ -238,16 +220,13 @@ func (s *SwarmService) GetSwarmUnlockKey(ctx context.Context) (*swarmtypes.Swarm
}
func (s *SwarmService) GetSwarmJoinTokens(ctx context.Context) (*swarmtypes.SwarmJoinTokensResponse, error) {
- if err := s.ensureSwarmManagerInternal(ctx); err != nil {
+ if err := s.ensureSwarmManager(ctx); err != nil {
return nil, err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- infoResult, err := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {
+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect swarm: %w", err)
}
@@ -259,16 +238,13 @@ func (s *SwarmService) GetSwarmJoinTokens(ctx context.Context) (*swarmtypes.Swar
}
func (s *SwarmService) RotateSwarmJoinTokens(ctx context.Context, req swarmtypes.SwarmRotateJoinTokensRequest) error {
- if err := s.ensureSwarmManagerInternal(ctx); err != nil {
+ if err := s.ensureSwarmManager(ctx); err != nil {
return err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- infoResult, err := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {
+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ })
if err != nil {
return fmt.Errorf("failed to inspect swarm: %w", err)
}
@@ -280,11 +256,13 @@ func (s *SwarmService) RotateSwarmJoinTokens(ctx context.Context, req swarmtypes
rotateManager = true
}
- if _, swarmUpdateErr := dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{
- Version: infoResult.Swarm.Version,
- Spec: infoResult.Swarm.Spec,
- RotateWorkerToken: rotateWorker,
- RotateManagerToken: rotateManager,
+ if _, swarmUpdateErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmUpdateResult, error) {
+ return dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{
+ Version: infoResult.Swarm.Version,
+ Spec: infoResult.Swarm.Spec,
+ RotateWorkerToken: rotateWorker,
+ RotateManagerToken: rotateManager,
+ })
}); swarmUpdateErr != nil {
return fmt.Errorf("failed to rotate swarm join tokens: %w", swarmUpdateErr)
}
@@ -293,35 +271,34 @@ func (s *SwarmService) RotateSwarmJoinTokens(ctx context.Context, req swarmtypes
}
func (s *SwarmService) UpdateSwarmSpec(ctx context.Context, req swarmtypes.SwarmUpdateRequest) error {
- if err := s.ensureSwarmManagerInternal(ctx); err != nil {
+ if err := s.ensureSwarmManager(ctx); err != nil {
return err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
version := req.Version
if version == 0 {
- infoResult, swarmInspectErr := dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ infoResult, swarmInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmInspectResult, error) {
+ return dockerClient.SwarmInspect(ctx, client.SwarmInspectOptions{})
+ })
if swarmInspectErr != nil {
return fmt.Errorf("failed to inspect swarm: %w", swarmInspectErr)
}
version = infoResult.Swarm.Version.Index
}
- spec, err := decodeSwarmSpecInternal(req.Spec)
+ spec, err := decodeSwarmSpec(req.Spec)
if err != nil {
return err
}
- if _, swarmUpdateErr := dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{
- Version: swarm.Version{Index: version},
- Spec: spec,
- RotateWorkerToken: req.RotateWorkerToken,
- RotateManagerToken: req.RotateManagerToken,
- RotateManagerUnlockKey: req.RotateManagerUnlockKey,
+ if _, swarmUpdateErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SwarmUpdateResult, error) {
+ return dockerClient.SwarmUpdate(ctx, client.SwarmUpdateOptions{
+ Version: swarm.Version{Index: version},
+ Spec: spec,
+ RotateWorkerToken: req.RotateWorkerToken,
+ RotateManagerToken: req.RotateManagerToken,
+ RotateManagerUnlockKey: req.RotateManagerUnlockKey,
+ })
}); swarmUpdateErr != nil {
return fmt.Errorf("failed to update swarm spec: %w", swarmUpdateErr)
}
@@ -329,13 +306,10 @@ func (s *SwarmService) UpdateSwarmSpec(ctx context.Context, req swarmtypes.Swarm
return nil
}
-func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- servicesResult, err := dockerClient.ServiceList(ctx, client.ServiceListOptions{})
+func (s *SwarmService) listTasksPaginatedWithFilters(ctx context.Context, filters client.Filters, params pagination.QueryParams) ([]swarmtypes.TaskSummary, pagination.Response, error) {
+ servicesResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ServiceListResult, error) {
+ return dockerClient.ServiceList(ctx, client.ServiceListOptions{})
+ })
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list swarm services: %w", err)
}
@@ -345,7 +319,9 @@ func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Context
serviceNameByID[service.ID] = service.Spec.Name
}
- nodesResult, err := dockerClient.NodeList(ctx, client.NodeListOptions{})
+ nodesResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.NodeListResult, error) {
+ return dockerClient.NodeList(ctx, client.NodeListOptions{})
+ })
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list swarm nodes: %w", err)
}
@@ -358,7 +334,9 @@ func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Context
if filters == nil {
filters = make(client.Filters)
}
- tasksResult, err := dockerClient.TaskList(ctx, client.TaskListOptions{Filters: filters})
+ tasksResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.TaskListResult, error) {
+ return dockerClient.TaskList(ctx, client.TaskListOptions{Filters: filters})
+ })
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to list swarm tasks: %w", err)
}
@@ -368,13 +346,28 @@ func (s *SwarmService) listTasksPaginatedWithFiltersInternal(ctx context.Context
items = append(items, swarmtypes.NewTaskSummary(task, serviceNameByID[task.ServiceID], nodeNameByID[task.NodeID]))
}
- config := s.buildTaskPaginationConfigInternal()
+ config := pagination.Config[swarmtypes.TaskSummary]{
+ SearchAccessors: []pagination.SearchAccessor[swarmtypes.TaskSummary]{
+ func(task swarmtypes.TaskSummary) (string, error) { return task.Name, nil },
+ func(task swarmtypes.TaskSummary) (string, error) { return task.ServiceName, nil },
+ func(task swarmtypes.TaskSummary) (string, error) { return task.NodeName, nil },
+ func(task swarmtypes.TaskSummary) (string, error) { return task.ID, nil },
+ func(task swarmtypes.TaskSummary) (string, error) { return task.CurrentState, nil },
+ },
+ SortBindings: []pagination.SortBinding[swarmtypes.TaskSummary]{
+ {Key: "service", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.ServiceName, b.ServiceName) }},
+ {Key: "node", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.NodeName, b.NodeName) }},
+ {Key: "state", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.CurrentState, b.CurrentState) }},
+ {Key: "created", Fn: func(a, b swarmtypes.TaskSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},
+ {Key: "updated", Fn: func(a, b swarmtypes.TaskSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},
+ },
+ }
result := config.SearchOrderAndPaginate(items, params)
paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, params)
return result.Items, paginationResp, nil
}
-func decodeSwarmSpecInternal(raw stdjson.RawMessage) (swarm.Spec, error) {
+func decodeSwarmSpec(raw stdjson.RawMessage) (swarm.Spec, error) {
trimmed := bytes.TrimSpace(raw)
if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) {
return swarm.Spec{}, errors.New("swarm spec is required")
@@ -392,20 +385,11 @@ func decodeSwarmSpecInternal(raw stdjson.RawMessage) (swarm.Spec, error) {
return spec, nil
}
-func defaultSwarmListenAddrInternal(listenAddr string) string {
- trimmed := strings.TrimSpace(listenAddr)
- return kit.Ternary(trimmed == "", defaultSwarmListenAddr, trimmed)
-}
-
-func (s *SwarmService) ensureSwarmManagerInternal(ctx context.Context) error {
- info, err := s.getDockerInfoInternal(ctx)
+func (s *SwarmService) ensureSwarmManager(ctx context.Context) error {
+ info, err := s.ensureSwarmActive(ctx)
if err != nil {
return err
}
-
- if info.Swarm.LocalNodeState != swarm.LocalNodeStateActive {
- return common.Classify(common.ErrSwarmNotEnabled, errors.New("Swarm mode is not enabled")) //nolint:staticcheck // Preserve the existing error message.
- }
if !info.Swarm.ControlAvailable {
return common.Classify(common.ErrSwarmManagerRequired, errors.New("Swarm manager access required")) //nolint:staticcheck // Preserve the existing error message.
}
@@ -413,26 +397,21 @@ func (s *SwarmService) ensureSwarmManagerInternal(ctx context.Context) error {
return nil
}
-func (s *SwarmService) ensureSwarmActiveInternal(ctx context.Context) error {
- info, err := s.getDockerInfoInternal(ctx)
+func (s *SwarmService) ensureSwarmActive(ctx context.Context) (system.Info, error) {
+ info, err := s.getDockerInfo(ctx)
if err != nil {
- return err
+ return system.Info{}, err
}
-
if info.Swarm.LocalNodeState != swarm.LocalNodeStateActive {
- return common.Classify(common.ErrSwarmNotEnabled, errors.New("Swarm mode is not enabled")) //nolint:staticcheck // Preserve the existing error message.
+ return system.Info{}, common.Classify(common.ErrSwarmNotEnabled, errors.New("Swarm mode is not enabled")) //nolint:staticcheck // Preserve the existing error message.
}
-
- return nil
+ return info, nil
}
-func (s *SwarmService) getDockerInfoInternal(ctx context.Context) (system.Info, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return system.Info{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})
+func (s *SwarmService) getDockerInfo(ctx context.Context) (system.Info, error) {
+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.SystemInfoResult, error) {
+ return dockerClient.Info(ctx, client.InfoOptions{})
+ })
if err != nil {
return system.Info{}, fmt.Errorf("failed to get Docker info: %w", err)
}
@@ -441,7 +420,7 @@ func (s *SwarmService) getDockerInfoInternal(ctx context.Context) (system.Info,
}
func (s *SwarmService) SyncSwarmEnabledState(ctx context.Context) error {
- info, err := s.getDockerInfoInternal(ctx)
+ info, err := s.getDockerInfo(ctx)
if err != nil {
return err
}
@@ -458,7 +437,7 @@ func (s *SwarmService) SyncSwarmEnabledState(ctx context.Context) error {
return nil
}
-func (s *SwarmService) persistSwarmEnabledStateInternal(ctx context.Context, enabled bool) {
+func (s *SwarmService) persistSwarmEnabledState(ctx context.Context, enabled bool) {
if s.kvService == nil {
return
}
@@ -467,22 +446,3 @@ func (s *SwarmService) persistSwarmEnabledStateInternal(ctx context.Context, ena
slog.WarnContext(ctx, "Failed to persist swarm enabled state", "enabled", enabled, "error", err)
}
}
-
-func (s *SwarmService) buildTaskPaginationConfigInternal() pagination.Config[swarmtypes.TaskSummary] {
- return pagination.Config[swarmtypes.TaskSummary]{
- SearchAccessors: []pagination.SearchAccessor[swarmtypes.TaskSummary]{
- func(task swarmtypes.TaskSummary) (string, error) { return task.Name, nil },
- func(task swarmtypes.TaskSummary) (string, error) { return task.ServiceName, nil },
- func(task swarmtypes.TaskSummary) (string, error) { return task.NodeName, nil },
- func(task swarmtypes.TaskSummary) (string, error) { return task.ID, nil },
- func(task swarmtypes.TaskSummary) (string, error) { return task.CurrentState, nil },
- },
- SortBindings: []pagination.SortBinding[swarmtypes.TaskSummary]{
- {Key: "service", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.ServiceName, b.ServiceName) }},
- {Key: "node", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.NodeName, b.NodeName) }},
- {Key: "state", Fn: func(a, b swarmtypes.TaskSummary) int { return strings.Compare(a.CurrentState, b.CurrentState) }},
- {Key: "created", Fn: func(a, b swarmtypes.TaskSummary) int { return a.CreatedAt.Compare(b.CreatedAt) }},
- {Key: "updated", Fn: func(a, b swarmtypes.TaskSummary) int { return a.UpdatedAt.Compare(b.UpdatedAt) }},
- },
- }
-}
diff --git a/backend/internal/swarm/service_test.go b/backend/internal/swarm/service_test.go
index 4af3cf479..7f1365cf5 100644
--- a/backend/internal/swarm/service_test.go
+++ b/backend/internal/swarm/service_test.go
@@ -7,20 +7,14 @@ import (
"github.com/stretchr/testify/require"
)
-func TestDecodeSwarmSpecInternal_AllowsEmptyObject(t *testing.T) {
- spec, err := decodeSwarmSpecInternal(json.RawMessage(`{}`))
+func TestDecodeSwarmSpec_AllowsEmptyObject(t *testing.T) {
+ spec, err := decodeSwarmSpec(json.RawMessage(`{}`))
require.NoError(t, err)
require.NotNil(t, spec.Labels)
require.Empty(t, spec.Labels)
}
-func TestDecodeSwarmSpecInternal_RejectsNull(t *testing.T) {
- _, err := decodeSwarmSpecInternal(json.RawMessage(`null`))
+func TestDecodeSwarmSpec_RejectsNull(t *testing.T) {
+ _, err := decodeSwarmSpec(json.RawMessage(`null`))
require.EqualError(t, err, "swarm spec is required")
}
-
-func TestDefaultSwarmListenAddrInternal(t *testing.T) {
- require.Equal(t, defaultSwarmListenAddr, defaultSwarmListenAddrInternal(""))
- require.Equal(t, defaultSwarmListenAddr, defaultSwarmListenAddrInternal(" "))
- require.Equal(t, "eth0:2377", defaultSwarmListenAddrInternal(" eth0:2377 "))
-}
diff --git a/backend/internal/system/children/backup/service.go b/backend/internal/system/children/backup/service.go
index cb3522171..1403bdd09 100644
--- a/backend/internal/system/children/backup/service.go
+++ b/backend/internal/system/children/backup/service.go
@@ -1041,7 +1041,9 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
if err != nil {
return errors.New("arcane system restore requires Arcane to run in Docker")
}
- inspectResult, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ inspectResult, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerID, client.ContainerInspectOptions{})
+ })
if err != nil {
return fmt.Errorf("inspect Arcane container: %w", err)
}
@@ -1153,28 +1155,34 @@ func (s *Service) RestoreBackup(ctx context.Context, id, recoveryKey string, use
ClusterOptions: executableMount.ClusterOptions,
})
}
- created, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: &container.Config{
- Image: current.Config.Image,
- Cmd: []string{kit.Ternary(executableMount != nil, systemRecoveryHelperPath, executablePath), "recovery-restore", "--request", requestFile},
- User: "0:0",
- Env: containerEnv,
- Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},
- },
- HostConfig: &container.HostConfig{
- AutoRemove: true,
- Mounts: mounts,
- NetworkMode: networkMode,
- SecurityOpt: slices.Clone(kit.FromPtr(current.HostConfig).SecurityOpt),
- Privileged: kit.FromPtr(current.HostConfig).Privileged,
- },
- Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),
+ created, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {
+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &container.Config{
+ Image: current.Config.Image,
+ Cmd: []string{kit.Ternary(executableMount != nil, systemRecoveryHelperPath, executablePath), "recovery-restore", "--request", requestFile},
+ User: "0:0",
+ Env: containerEnv,
+ Labels: map[string]string{"com.getarcaneapp.arcane.recovery": "true", "com.getarcaneapp.arcane": "true"},
+ },
+ HostConfig: &container.HostConfig{
+ AutoRemove: true,
+ Mounts: mounts,
+ NetworkMode: networkMode,
+ SecurityOpt: slices.Clone(kit.FromPtr(current.HostConfig).SecurityOpt),
+ Privileged: kit.FromPtr(current.HostConfig).Privileged,
+ },
+ Name: fmt.Sprintf("%s-recovery-%d", strings.TrimPrefix(current.Name, "/"), time.Now().Unix()),
+ })
})
if err != nil {
return fmt.Errorf("create recovery helper container: %w", err)
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); containerStartErr != nil {
- _, _ = dockerClient.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {
+ return cli.ContainerStart(ctx, created.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {
+ return cli.ContainerRemove(ctx, created.ID, client.ContainerRemoveOptions{Force: true})
+ })
return fmt.Errorf("start recovery helper container: %w", containerStartErr)
}
cleanupRequest = false
diff --git a/backend/internal/system/children/prune/service.go b/backend/internal/system/children/prune/service.go
index 2b0ea382b..adbc25456 100644
--- a/backend/internal/system/children/prune/service.go
+++ b/backend/internal/system/children/prune/service.go
@@ -26,6 +26,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/imageupdate"
"github.com/getarcaneapp/arcane/backend/v2/internal/network"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
@@ -38,7 +39,7 @@ import (
// Service runs manual and scheduled prunes, serializing them per environment.
type Service struct {
location *time.Location
- dockerClient func(ctx context.Context) (*client.Client, error)
+ dockerService *docker.DockerClientService
activityService *activity.ActivityService
imageUpdateService *imageupdate.ImageUpdateService
volumeService *volume.VolumeService
@@ -49,7 +50,7 @@ type Service struct {
func NewService(
location *time.Location,
- dockerClient func(ctx context.Context) (*client.Client, error),
+ dockerService *docker.DockerClientService,
activityService *activity.ActivityService,
imageUpdateService *imageupdate.ImageUpdateService,
volumeService *volume.VolumeService,
@@ -57,7 +58,7 @@ func NewService(
) *Service {
return &Service{
location: location,
- dockerClient: dockerClient,
+ dockerService: dockerService,
activityService: activityService,
imageUpdateService: imageUpdateService,
volumeService: volumeService,
@@ -340,11 +341,6 @@ func (s *Service) completeSystemPruneActivity(ctx context.Context, activityID st
}
func (s *Service) pruneContainers(ctx context.Context, options system.PruneContainersOptions) (system.PruneAllResult, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
filterArgs := make(client.Filters)
if options.Mode == system.PruneContainerModeOlderThan {
if strings.TrimSpace(options.Until) == "" {
@@ -353,7 +349,11 @@ func (s *Service) pruneContainers(ctx context.Context, options system.PruneConta
filterArgs = filterArgs.Add("until", options.Until)
}
- report, err := dockerClient.ContainerPrune(ctx, client.ContainerPruneOptions{Filters: filterArgs})
+ cli, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune containers: %w", err)
+ }
+ report, err := cli.ContainerPrune(ctx, client.ContainerPruneOptions{Filters: filterArgs})
if err != nil {
return system.PruneAllResult{}, fmt.Errorf("failed to prune containers: %w", err)
}
@@ -366,11 +366,6 @@ func (s *Service) pruneContainers(ctx context.Context, options system.PruneConta
}
func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOptions) (system.PruneAllResult, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
filterArgs := make(client.Filters)
switch options.Mode {
case system.PruneImageModeNone:
@@ -389,7 +384,11 @@ func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOpt
return system.PruneAllResult{}, fmt.Errorf("unsupported image prune mode: %s", options.Mode)
}
- report, err := dockerClient.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})
+ cli, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune images: %w", err)
+ }
+ report, err := cli.ImagePrune(ctx, client.ImagePruneOptions{Filters: filterArgs})
if err != nil {
return system.PruneAllResult{}, fmt.Errorf("failed to prune images: %w", err)
}
@@ -418,17 +417,17 @@ func (s *Service) pruneImages(ctx context.Context, options system.PruneImagesOpt
}
func (s *Service) pruneBuildCache(ctx context.Context, options system.PruneBuildCacheOptions) (system.PruneAllResult, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- slog.ErrorContext(ctx, "Error connecting to Docker for build cache prune", "error", err.Error())
- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker for build cache prune: %w", err)
- }
if options.Mode == system.PruneBuildCacheModeOlderThan && strings.TrimSpace(options.Until) == "" {
return system.PruneAllResult{}, errors.New("build cache prune mode olderThan requires until")
}
slog.DebugContext(ctx, "starting build cache pruning", "mode", options.Mode, "until", options.Until)
- report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{
+ cli, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ slog.ErrorContext(ctx, "Error pruning build cache", "error", err.Error())
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune build cache: %w", err)
+ }
+ report, err := cli.BuildCachePrune(ctx, client.BuildCachePruneOptions{
All: options.Mode == system.PruneBuildCacheModeAll,
Filters: kit.Ternary(options.Mode == system.PruneBuildCacheModeOlderThan, make(client.Filters).Add("until", options.Until), nil),
})
@@ -461,11 +460,6 @@ func (s *Service) pruneVolumes(ctx context.Context, options system.PruneVolumesO
}
func (s *Service) pruneNetworks(ctx context.Context, options system.PruneNetworksOptions) (system.PruneAllResult, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return system.PruneAllResult{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
filterArgs := make(client.Filters)
if options.Mode == system.PruneNetworkModeOlderThan {
if strings.TrimSpace(options.Until) == "" {
@@ -474,7 +468,11 @@ func (s *Service) pruneNetworks(ctx context.Context, options system.PruneNetwork
filterArgs = filterArgs.Add("until", options.Until)
}
- report, err := dockerClient.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: filterArgs})
+ cli, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return system.PruneAllResult{}, fmt.Errorf("failed to prune networks: %w", err)
+ }
+ report, err := cli.NetworkPrune(ctx, client.NetworkPruneOptions{Filters: filterArgs})
if err != nil {
return system.PruneAllResult{}, fmt.Errorf("failed to prune networks: %w", err)
}
@@ -546,19 +544,17 @@ func (s *Service) PruneScheduled(ctx context.Context, environmentID string, req
}
func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllRequest) (prunePlan, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return prunePlan{}, err
- }
if req.Volumes != nil {
s.volumeService.CleanupHelperContainers(ctx)
}
- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{
- Containers: true,
- Images: true,
- Volumes: true,
- BuildCache: req.BuildCache != nil,
- Verbose: true,
+ usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {
+ return cli.DiskUsage(ctx, client.DiskUsageOptions{
+ Containers: true,
+ Images: true,
+ Volumes: true,
+ BuildCache: req.BuildCache != nil,
+ Verbose: true,
+ })
})
if err != nil {
return prunePlan{}, err
@@ -572,7 +568,7 @@ func (s *Service) preparePrunePlan(ctx context.Context, req system.PruneAllReque
if err != nil {
return prunePlan{}, err
}
- networks, err := s.selectPruneNetworks(ctx, dockerClient, req, selectedContainers)
+ networks, err := s.selectPruneNetworks(ctx, req, selectedContainers)
if err != nil {
return prunePlan{}, err
}
@@ -656,7 +652,7 @@ func (s *Service) selectPruneVolumes(req system.PruneAllRequest, usage client.Di
return resources
}
-func (s *Service) selectPruneNetworks(ctx context.Context, dockerClient *client.Client, req system.PruneAllRequest, selectedContainers map[string]bool) ([]pruneResource, error) {
+func (s *Service) selectPruneNetworks(ctx context.Context, req system.PruneAllRequest, selectedContainers map[string]bool) ([]pruneResource, error) {
if req.Networks == nil || req.Networks.Mode == system.PruneNetworkModeNone {
return nil, nil
}
@@ -664,7 +660,9 @@ func (s *Service) selectPruneNetworks(ctx context.Context, dockerClient *client.
if err != nil {
return nil, err
}
- networks, err := dockerClient.NetworkList(ctx, client.NetworkListOptions{})
+ networks, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NetworkListResult, error) {
+ return cli.NetworkList(ctx, client.NetworkListOptions{})
+ })
if err != nil {
return nil, err
}
@@ -674,7 +672,9 @@ nextNetwork:
if item.Name == "bridge" || item.Name == "host" || item.Name == "none" || item.Ingress || item.Scope != "local" || (!until.IsZero() && !item.Created.Before(until)) {
continue
}
- inspect, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, item.ID, client.NetworkInspectOptions{})
+ inspect, networkInspectWithCompatibilityErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NetworkInspectResult, error) {
+ return compat.NetworkInspectWithCompatibility(ctx, cli, item.ID, client.NetworkInspectOptions{})
+ })
if networkInspectWithCompatibilityErr != nil {
return nil, networkInspectWithCompatibilityErr
}
@@ -778,10 +778,6 @@ func (s *Service) executePrunePlan(
ActivityID: mo.EmptyableToOption(activityID).ToPointer(),
}, err
}
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return finish(err)
- }
for _, resource := range plan.Resources {
if resource.Kind == "build_cache" {
continue
@@ -799,7 +795,7 @@ func (s *Service) executePrunePlan(
}); progressErr != nil {
return finish(progressErr)
}
- size, eligible, exists, eligibleErr := s.pruneResourceEligible(ctx, dockerClient, plan, resource)
+ size, eligible, exists, eligibleErr := s.pruneResourceEligible(ctx, plan, resource)
if eligibleErr != nil {
return finish(eligibleErr)
}
@@ -807,7 +803,7 @@ func (s *Service) executePrunePlan(
if !exists || !eligible {
return nil
}
- return s.removePruneResource(ctx, dockerClient, resource)
+ return s.removePruneResource(ctx, resource)
}()
removeFailed := removeErr != nil && !errdefs.IsNotFound(removeErr)
if removeFailed {
@@ -856,20 +852,24 @@ func (s *Service) executePrunePlan(
return finish(deleteRecordsForImagesErr)
}
}
- reclaimed, err := s.executePruneCache(ctx, dockerClient, plan, recovery)
+ reclaimed, err := s.executePruneCache(ctx, plan, recovery)
spaceReclaimed += reclaimed
buildCacheSpace += reclaimed
return finish(err)
}
-func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.Client, resource pruneResource) error {
+func (s *Service) removePruneResource(ctx context.Context, resource pruneResource) error {
switch resource.Kind {
case "container":
- _, err := dockerClient.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {
+ return cli.ContainerRemove(ctx, resource.ID, client.ContainerRemoveOptions{})
+ })
return err
case "image":
for _, name := range resource.Names {
- inspected, inspectErr := dockerClient.ImageInspect(ctx, name)
+ inspected, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {
+ return cli.ImageInspect(ctx, name)
+ })
if errdefs.IsNotFound(inspectErr) {
continue
}
@@ -879,11 +879,15 @@ func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.
if inspected.ID != resource.ID {
continue
}
- if _, err := dockerClient.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false}); err != nil {
+ if _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageRemoveResult, error) {
+ return cli.ImageRemove(ctx, name, client.ImageRemoveOptions{PruneChildren: false})
+ }); err != nil {
return err
}
}
- _, err := dockerClient.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageRemoveResult, error) {
+ return cli.ImageRemove(ctx, resource.ID, client.ImageRemoveOptions{PruneChildren: false})
+ })
return err
case "volume":
return s.volumeService.DeleteVolume(ctx, resource.ID, false, user.SystemUser)
@@ -894,13 +898,15 @@ func (s *Service) removePruneResource(ctx context.Context, dockerClient *client.
}
}
-func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Client, plan prunePlan, recovery bool) (uint64, error) {
+func (s *Service) executePruneCache(ctx context.Context, plan prunePlan, recovery bool) (uint64, error) {
request := plan.Request.BuildCache
if request == nil || request.Mode == system.PruneBuildCacheModeNone {
return 0, nil
}
if recovery {
- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})
+ usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {
+ return cli.DiskUsage(ctx, client.DiskUsageOptions{BuildCache: true, Verbose: true})
+ })
if err != nil {
return 0, err
}
@@ -927,7 +933,11 @@ func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Cl
if request.Mode == system.PruneBuildCacheModeOlderThan {
filters = filters.Add("until", request.Until)
}
- report, err := dockerClient.BuildCachePrune(ctx, client.BuildCachePruneOptions{
+ cli, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return reclaimed, err
+ }
+ report, err := cli.BuildCachePrune(ctx, client.BuildCachePruneOptions{
All: request.Mode == system.PruneBuildCacheModeAll,
Filters: filters,
})
@@ -940,9 +950,11 @@ func (s *Service) executePruneCache(ctx context.Context, dockerClient *client.Cl
}
// pruneResourceEligible re-checks a planned resource against live state and returns its size from that snapshot.
-func (s *Service) pruneResourceEligible(ctx context.Context, dockerClient *client.Client, plan prunePlan, resource pruneResource) (size int64, eligible, exists bool, err error) {
+func (s *Service) pruneResourceEligible(ctx context.Context, plan prunePlan, resource pruneResource) (size int64, eligible, exists bool, err error) {
if resource.Kind == "network" {
- inspected, networkInspectWithCompatibilityErr := compat.NetworkInspectWithCompatibility(ctx, dockerClient, resource.ID, client.NetworkInspectOptions{})
+ inspected, networkInspectWithCompatibilityErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.NetworkInspectResult, error) {
+ return compat.NetworkInspectWithCompatibility(ctx, cli, resource.ID, client.NetworkInspectOptions{})
+ })
if errdefs.IsNotFound(networkInspectWithCompatibilityErr) {
return resource.Size, false, false, nil
}
@@ -951,11 +963,13 @@ func (s *Service) pruneResourceEligible(ctx context.Context, dockerClient *clien
}
return resource.Size, len(inspected.Network.Containers) == 0 && !inspected.Network.Ingress, true, nil
}
- usage, err := dockerClient.DiskUsage(ctx, client.DiskUsageOptions{
- Containers: resource.Kind == "container",
- Images: resource.Kind == "image",
- Volumes: resource.Kind == "volume",
- Verbose: true,
+ usage, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.DiskUsageResult, error) {
+ return cli.DiskUsage(ctx, client.DiskUsageOptions{
+ Containers: resource.Kind == "container",
+ Images: resource.Kind == "image",
+ Volumes: resource.Kind == "volume",
+ Verbose: true,
+ })
})
if err != nil {
return resource.Size, false, false, err
diff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.go
index 1c2a0644b..faf16bbf1 100644
--- a/backend/internal/system/children/upgrade/service.go
+++ b/backend/internal/system/children/upgrade/service.go
@@ -262,12 +262,16 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targ
}
return *self, nil
}
- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ inspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerID, client.ContainerInspectOptions{})
+ })
if inspectErr == nil {
return inspect.Container, nil
}
// Fall back to a prefix match across every container.
- containers, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ containers, listErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerListResult, error) {
+ return cli.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if listErr != nil {
return container.InspectResponse{}, fmt.Errorf("inspect container: %w", listErr)
}
@@ -275,7 +279,9 @@ func (s *Service) prepareUpgrade(ctx context.Context, user usertypes.Actor, targ
if idx < 0 {
return container.InspectResponse{}, fmt.Errorf("inspect container: %w", common.Classify(common.ErrNotFound, errors.New("could not find Arcane container")))
}
- if inspect, inspectErr = compat.ContainerInspectWithCompatibility(ctx, dockerClient, containers.Items[idx].ID, client.ContainerInspectOptions{}); inspectErr != nil {
+ if inspect, inspectErr = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, cli, containers.Items[idx].ID, client.ContainerInspectOptions{})
+ }); inspectErr != nil {
return container.InspectResponse{}, fmt.Errorf("inspect container: %w", inspectErr)
}
return inspect.Container, nil
@@ -409,7 +415,9 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgr
// On SELinux-enforcing hosts the socket is container_var_run_t, so without a label opt the upgrader exits with EACCES.
if !kit.FromPtr(prepared.current.HostConfig).Privileged &&
!slices.ContainsFunc(securityOpt, func(opt string) bool { return strings.HasPrefix(strings.TrimSpace(opt), "label") }) {
- infoResult, infoErr := dockerClient.Info(ctx, client.InfoOptions{})
+ infoResult, infoErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.SystemInfoResult, error) {
+ return cli.Info(ctx, client.InfoOptions{})
+ })
if infoErr != nil {
slog.DebugContext(ctx, "Failed to query daemon info for SELinux detection", "error", infoErr)
} else if slices.Contains(infoResult.Info.SecurityOptions, "name=selinux") {
@@ -418,43 +426,49 @@ func (s *Service) runPreparedUpgrade(ctx context.Context, prepared *preparedUpgr
}
upgraderName := fmt.Sprintf("%s-upgrader-%d", prepared.containerName, time.Now().Unix())
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: &container.Config{
- Image: prepared.pullImage,
- Cmd: upgradeCmd,
- // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.
- User: "0:0",
- Env: containerEnv,
- Labels: map[string]string{
- "com.getarcaneapp.arcane.upgrader": "true",
- "com.getarcaneapp.arcane": "true",
- },
- },
- HostConfig: &container.HostConfig{
- AutoRemove: !keepUpgraderContainer,
- Mounts: mounts,
- NetworkMode: networkMode,
- SecurityOpt: securityOpt,
- Privileged: kit.FromPtr(prepared.current.HostConfig).Privileged,
- },
- NetworkingConfig: func() *network.NetworkingConfig {
- if networkMode == "" || !networkMode.IsUserDefined() {
- return nil
- }
- return &network.NetworkingConfig{
- EndpointsConfig: map[string]*network.EndpointSettings{
- string(networkMode): {Aliases: []string{"arcane-upgrader"}},
+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {
+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &container.Config{
+ Image: prepared.pullImage,
+ Cmd: upgradeCmd,
+ // Root for the Docker socket: the short-lived upgrader never goes through the runtime-identity drop.
+ User: "0:0",
+ Env: containerEnv,
+ Labels: map[string]string{
+ "com.getarcaneapp.arcane.upgrader": "true",
+ "com.getarcaneapp.arcane": "true",
},
- }
- }(),
- Name: upgraderName,
+ },
+ HostConfig: &container.HostConfig{
+ AutoRemove: !keepUpgraderContainer,
+ Mounts: mounts,
+ NetworkMode: networkMode,
+ SecurityOpt: securityOpt,
+ Privileged: kit.FromPtr(prepared.current.HostConfig).Privileged,
+ },
+ NetworkingConfig: func() *network.NetworkingConfig {
+ if networkMode == "" || !networkMode.IsUserDefined() {
+ return nil
+ }
+ return &network.NetworkingConfig{
+ EndpointsConfig: map[string]*network.EndpointSettings{
+ string(networkMode): {Aliases: []string{"arcane-upgrader"}},
+ },
+ }
+ }(),
+ Name: upgraderName,
+ })
})
if err != nil {
return "", fmt.Errorf("create upgrader container: %w", err)
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {
+ return cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {
+ return cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ })
return "", fmt.Errorf("start upgrader container: %w", containerStartErr)
}
diff --git a/backend/internal/system/handler.go b/backend/internal/system/handler.go
index b479d584f..f94820050 100644
--- a/backend/internal/system/handler.go
+++ b/backend/internal/system/handler.go
@@ -1,7 +1,6 @@
package system
import (
- "cmp"
"context"
"errors"
"strings"
@@ -11,20 +10,15 @@ import (
"github.com/getarcaneapp/arcane/types/v2/container"
"github.com/getarcaneapp/arcane/types/v2/dockerinfo"
"github.com/getarcaneapp/arcane/types/v2/system"
- dockersystem "github.com/moby/moby/api/types/system"
- "github.com/moby/moby/client"
"go.getarcane.app/docker/convert"
"go.getarcane.app/docker/convert/types"
- "go.getarcane.app/sys/cgroup"
- "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"
)
// SystemHandler handles system management endpoints.
type SystemHandler struct {
- dockerService *docker.DockerClientService
systemService *SystemService
appCtx context.Context
}
@@ -62,19 +56,13 @@ type ConvertDockerRunOutput struct {
Body system.ConvertDockerRunResponse
}
-func NewHandler(dockerService *docker.DockerClientService, systemService *SystemService, appCtx context.Context) *SystemHandler {
- return &SystemHandler{dockerService: dockerService, systemService: systemService, appCtx: appCtx}
+func NewHandler(systemService *SystemService, appCtx context.Context) *SystemHandler {
+ return &SystemHandler{systemService: systemService, appCtx: appCtx}
}
// Health checks if the Docker daemon is responsive.
func (h *SystemHandler) Health(ctx context.Context, input *SystemHealthInput) (*struct{}, error) {
- dockerClient, err := h.dockerService.GetClient(ctx)
- if err != nil {
- return nil, huma.Error503ServiceUnavailable("Failed to connect to Docker: " + err.Error())
- }
-
- _, err = dockerClient.Ping(ctx, client.PingOptions{})
- if err != nil {
+ if err := h.systemService.PingDocker(ctx); err != nil {
return nil, huma.Error503ServiceUnavailable("Docker is not responsive: " + err.Error())
}
@@ -83,75 +71,12 @@ func (h *SystemHandler) Health(ctx context.Context, input *SystemHealthInput) (*
// GetDockerInfo returns Docker daemon version and system information.
func (h *SystemHandler) GetDockerInfo(ctx context.Context, input *GetDockerInfoInput) (*GetDockerInfoOutput, error) {
- dockerClient, err := h.dockerService.GetClient(ctx)
- if err != nil {
- return nil, huma.Error500InternalServerError("Failed to connect to Docker: " + err.Error())
- }
-
- version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})
- if err != nil {
- return nil, huma.Error500InternalServerError("Failed to get Docker version: " + err.Error())
- }
-
- infoResult, err := dockerClient.Info(ctx, client.InfoOptions{})
+ info, err := h.systemService.GetDockerInfo(ctx)
if err != nil {
return nil, huma.Error500InternalServerError("Failed to get Docker info: " + err.Error())
}
- info := infoResult.Info
-
- // Apply cgroup limits only when running outside Docker (e.g. in LXC).
- // In Docker, --cpus/--memory are artificial operator constraints that
- // should not cap the host totals shown in the dashboard. The Docker
- // daemon's NCPU/MemTotal already reflect the real host. In LXC the
- // daemon may report the physical machine's full capacity while the
- // LXC guest has a smaller cgroup budget — apply those limits so the
- // dashboard shows what Arcane's host actually has available.
- if !cgroup.IsDockerContainer() {
- if cgroupLimits, detectLimitsErr := cgroup.DetectLimits(); detectLimitsErr == nil {
- if limit := cgroupLimits.MemoryLimit; limit > 0 && (info.MemTotal == 0 || limit < info.MemTotal) {
- info.MemTotal = limit // keep: copying Docker's ~70-field system.Info as a literal is unreadable
- }
- if cgroupLimits.CPUCount > 0 && (info.NCPU == 0 || cgroupLimits.CPUCount < info.NCPU) {
- info.NCPU = cgroupLimits.CPUCount // keep: copying Docker's ~70-field system.Info as a literal is unreadable
- }
- }
- }
-
- gitCommit, goVersion, buildTime := extractVersionDetailsFromComponents(version.Components)
-
- return &GetDockerInfoOutput{
- Body: dockerinfo.Info{
- Success: true,
- APIVersion: version.APIVersion,
- GitCommit: gitCommit,
- GoVersion: goVersion,
- Os: version.Os,
- Arch: version.Arch,
- BuildTime: buildTime,
- Info: info,
- },
- }, nil
-}
-
-func extractVersionDetailsFromComponents(components []dockersystem.ComponentVersion) (gitCommit, goVersion, buildTime string) {
- for _, component := range components {
- if component.Details == nil {
- continue
- }
-
- for key, value := range component.Details {
- switch strings.ToLower(key) {
- case "gitcommit":
- gitCommit = cmp.Or(gitCommit, value)
- case "goversion":
- goVersion = cmp.Or(goVersion, value)
- case "buildtime":
- buildTime = cmp.Or(buildTime, value)
- }
- }
- }
- return gitCommit, goVersion, buildTime
+ return &GetDockerInfoOutput{Body: info}, nil
}
// StartAllContainers starts all Docker containers.
diff --git a/backend/internal/system/module.go b/backend/internal/system/module.go
index f3eb46dd9..65d7e1051 100644
--- a/backend/internal/system/module.go
+++ b/backend/internal/system/module.go
@@ -10,7 +10,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/activity"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
- "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
"github.com/getarcaneapp/arcane/backend/v2/internal/system/children/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/system/children/prune"
@@ -22,7 +21,6 @@ import (
// Module wires the system domain and mounts its routes.
type Module struct {
service *SystemService
- dockerService *docker.DockerClientService
activityService *activity.ActivityService
cfg *config.Config
}
@@ -30,11 +28,10 @@ type Module struct {
// New wires system routes around an existing service.
func New(
service *SystemService,
- dockerService *docker.DockerClientService,
activityService *activity.ActivityService,
cfg *config.Config,
) *Module {
- return &Module{service: service, dockerService: dockerService, activityService: activityService, cfg: cfg}
+ return &Module{service: service, activityService: activityService, cfg: cfg}
}
// Service exposes the system service to collaborators.
@@ -53,7 +50,7 @@ func (m *Module) RegisterRoutes(api huma.API, appCtx handlerutil.ActivityAppCont
}
service := cmp.Or(m.service, &SystemService{})
- RegisterSystem(api, NewHandler(m.dockerService, m.service, appCtx.Context()))
+ RegisterSystem(api, NewHandler(m.service, appCtx.Context()))
prune.RegisterRoutes(api, prune.NewHandler(service.prune, appCtx.Context()))
upgrade.RegisterRoutes(api, upgrade.NewHandler(service.upgrade, m.cfg, appCtx.Context()))
backup.RegisterRoutes(api, backup.NewHandler(service.backup, m.activityService, appCtx.Context()))
diff --git a/backend/internal/system/service.go b/backend/internal/system/service.go
index 287cc739e..119c6c4c4 100644
--- a/backend/internal/system/service.go
+++ b/backend/internal/system/service.go
@@ -14,6 +14,7 @@ import (
activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"
backuptypes "github.com/getarcaneapp/arcane/types/v2/backup"
containertypes "github.com/getarcaneapp/arcane/types/v2/container"
+ "github.com/getarcaneapp/arcane/types/v2/dockerinfo"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/system"
usertypes "github.com/getarcaneapp/arcane/types/v2/user"
@@ -100,7 +101,7 @@ func NewSystemService(
dockerHostMemoryCache: hot.NewHotCache[struct{}, dockerHostMemoryInfo](hot.LRU, 1).
WithTTL(dockerHostMemoryCacheTTL).
Build(),
- prune: prune.NewService(cfg.GetLocation(), dockerService.GetClient, activityService, imageUpdateService, volumeService, networkService),
+ prune: prune.NewService(cfg.GetLocation(), dockerService, activityService, imageUpdateService, volumeService, networkService),
upgrade: upgrade.NewService(db, dockerService, versionService, eventService, settingsService, projectService, resolveUpgraderRuntimeOptions),
}
s.backup = systembackup.NewService(systembackup.Dependencies{
@@ -302,6 +303,70 @@ func (s *SystemService) performBatchContainerAction(ctx context.Context, environ
return result, listErr
}
+// PingDocker checks that the Docker daemon is responsive.
+func (s *SystemService) PingDocker(ctx context.Context) error {
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.PingResult, error) {
+ return cli.Ping(ctx, client.PingOptions{})
+ })
+ return err
+}
+
+// GetDockerInfo returns Docker daemon version and system information.
+func (s *SystemService) GetDockerInfo(ctx context.Context) (dockerinfo.Info, error) {
+ serverVersion, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ServerVersionResult, error) {
+ return cli.ServerVersion(ctx, client.ServerVersionOptions{})
+ })
+ if err != nil {
+ return dockerinfo.Info{}, fmt.Errorf("failed to get Docker version: %w", err)
+ }
+
+ infoResult, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.SystemInfoResult, error) {
+ return cli.Info(ctx, client.InfoOptions{})
+ })
+ if err != nil {
+ return dockerinfo.Info{}, err
+ }
+ info := infoResult.Info
+
+ // Outside Docker (e.g. LXC) the daemon may report the whole machine, so cap totals at the guest's cgroup budget.
+ // Inside Docker, --cpus/--memory are operator limits and the daemon's totals already reflect the host.
+ if !cgroup.IsDockerContainer() {
+ if cgroupLimits, detectLimitsErr := cgroup.DetectLimits(); detectLimitsErr == nil {
+ if limit := cgroupLimits.MemoryLimit; limit > 0 && (info.MemTotal == 0 || limit < info.MemTotal) {
+ info.MemTotal = limit // keep: copying Docker's ~70-field system.Info as a literal is unreadable
+ }
+ if cgroupLimits.CPUCount > 0 && (info.NCPU == 0 || cgroupLimits.CPUCount < info.NCPU) {
+ info.NCPU = cgroupLimits.CPUCount // keep: copying Docker's ~70-field system.Info as a literal is unreadable
+ }
+ }
+ }
+
+ var gitCommit, goVersion, buildTime string
+ for _, component := range serverVersion.Components {
+ for key, value := range component.Details {
+ switch strings.ToLower(key) {
+ case "gitcommit":
+ gitCommit = cmp.Or(gitCommit, value)
+ case "goversion":
+ goVersion = cmp.Or(goVersion, value)
+ case "buildtime":
+ buildTime = cmp.Or(buildTime, value)
+ }
+ }
+ }
+
+ return dockerinfo.Info{
+ Success: true,
+ APIVersion: serverVersion.APIVersion,
+ GitCommit: gitCommit,
+ GoVersion: goVersion,
+ Os: serverVersion.Os,
+ Arch: serverVersion.Arch,
+ BuildTime: buildTime,
+ Info: info,
+ }, nil
+}
+
func (s *SystemService) GetDiskUsagePath(ctx context.Context) string {
return cmp.Or(s.settingsService.GetSettingsConfig().DiskUsagePath.Value, "/")
}
@@ -347,11 +412,9 @@ func (s *SystemService) loadDockerHostMemory(ctx context.Context) (dockerHostMem
if err != nil {
return dockerHostMemoryInfo{}, err
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return dockerHostMemoryInfo{}, err
- }
- inspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ inspect, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerID, client.ContainerInspectOptions{})
+ })
if err != nil {
return dockerHostMemoryInfo{}, err
}
@@ -364,7 +427,9 @@ func (s *SystemService) loadDockerHostMemory(ctx context.Context) (dockerHostMem
return dockerHostMemoryInfo{}, err
}
- info, err := dockerClient.Info(ctx, client.InfoOptions{})
+ info, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.SystemInfoResult, error) {
+ return cli.Info(ctx, client.InfoOptions{})
+ })
if err != nil {
return dockerHostMemoryInfo{}, err
}
@@ -423,26 +488,29 @@ const systemBackupHistoryUnion = "SELECT id, size, created_at, status, trigger,
// backupStore persists system backup runs and policies for the backup child.
func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) systembackup.Store {
+ toModel := func(dto *backuptypes.SystemBackupRun) SystemBackupRun {
+ return SystemBackupRun{ //exhaustruct:enforce
+ BaseModel: database.BaseModel{
+ ID: dto.ID,
+ CreatedAt: time.Time{},
+ UpdatedAt: nil,
+ },
+ Size: dto.Size,
+ CreatedAt: dto.CreatedAt,
+ Status: SystemBackupStatus(dto.Status),
+ Trigger: SystemBackupTrigger(dto.Trigger),
+ Destination: dto.Destination,
+ LocalSnapshotID: dto.LocalSnapshotID,
+ RemoteSnapshotID: dto.RemoteSnapshotID,
+ S3DestinationID: dto.S3DestinationID,
+ S3DestinationName: dto.S3DestinationName,
+ PolicyID: dto.PolicyID,
+ Error: dto.Error,
+ }
+ }
return systembackup.Store{
CreateRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {
- run := SystemBackupRun{ //exhaustruct:enforce
- BaseModel: database.BaseModel{
- ID: dto.ID,
- CreatedAt: time.Time{},
- UpdatedAt: nil,
- },
- Size: dto.Size,
- CreatedAt: dto.CreatedAt,
- Status: SystemBackupStatus(dto.Status),
- Trigger: SystemBackupTrigger(dto.Trigger),
- Destination: dto.Destination,
- LocalSnapshotID: dto.LocalSnapshotID,
- RemoteSnapshotID: dto.RemoteSnapshotID,
- S3DestinationID: dto.S3DestinationID,
- S3DestinationName: dto.S3DestinationName,
- PolicyID: dto.PolicyID,
- Error: dto.Error,
- }
+ run := toModel(dto)
if err := s.db.WithContext(ctx).Create(&run).Error; err != nil {
return err
}
@@ -465,24 +533,7 @@ func (s *SystemService) backupStore(s3Destinations *s3.S3DestinationService) sys
return nil
},
SaveRun: func(ctx context.Context, dto *backuptypes.SystemBackupRun) error {
- return s.db.WithContext(ctx).Save(&SystemBackupRun{ //exhaustruct:enforce
- BaseModel: database.BaseModel{
- ID: dto.ID,
- CreatedAt: time.Time{},
- UpdatedAt: nil,
- },
- Size: dto.Size,
- CreatedAt: dto.CreatedAt,
- Status: SystemBackupStatus(dto.Status),
- Trigger: SystemBackupTrigger(dto.Trigger),
- Destination: dto.Destination,
- LocalSnapshotID: dto.LocalSnapshotID,
- RemoteSnapshotID: dto.RemoteSnapshotID,
- S3DestinationID: dto.S3DestinationID,
- S3DestinationName: dto.S3DestinationName,
- PolicyID: dto.PolicyID,
- Error: dto.Error,
- }).Error
+ return s.db.WithContext(ctx).Save(new(toModel(dto))).Error
},
FailRun: func(ctx context.Context, id, message string) error {
return s.db.WithContext(ctx).Model(&SystemBackupRun{}).Where("id = ?", id).Updates(map[string]any{"status": SystemBackupStatusFailed, "error": message}).Error
diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.go
index 6870953a7..87fc69290 100644
--- a/backend/internal/updater/children/recovery/service.go
+++ b/backend/internal/updater/children/recovery/service.go
@@ -8,6 +8,7 @@ import (
"slices"
"strings"
+ "github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
"github.com/moby/moby/api/types/container"
@@ -17,6 +18,7 @@ import (
"go.getarcane.app/updater"
"go.getarcane.app/updater/refs"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
)
@@ -27,14 +29,19 @@ var errNoFrozenPlan = errors.New("interrupted update has no frozen target plan")
// Service freezes update targets before any pull, pins pulls to the frozen
// digests, and confirms or resumes interrupted updates.
type Service struct {
- dockerClient func(ctx context.Context) (*client.Client, error)
- digestResolver func() updater.RegistryDigestResolver
+ dockerService *docker.DockerClientService
+ images remoteDigester
pendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error)
activityID func(ctx context.Context) string
acquireUpdate func(ctx context.Context) (context.Context, func(), error)
applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error)
}
+// remoteDigester resolves registry digests; the image service implements it.
+type remoteDigester interface {
+ RemoteDigest(ctx context.Context, imageRef string, credentials []containerregistry.Credential) (string, error)
+}
+
type (
frozenPendingKey struct{}
frozenSingleKey struct{}
@@ -50,16 +57,16 @@ type frozenSingle struct {
}
func NewService(
- dockerClient func(ctx context.Context) (*client.Client, error),
- digestResolver func() updater.RegistryDigestResolver,
+ dockerService *docker.DockerClientService,
+ images remoteDigester,
pendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error),
activityID func(ctx context.Context) string,
acquireUpdate func(ctx context.Context) (context.Context, func(), error),
applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error),
) *Service {
return &Service{
- dockerClient: dockerClient,
- digestResolver: digestResolver,
+ dockerService: dockerService,
+ images: images,
pendingUpdates: pendingUpdates,
activityID: activityID,
acquireUpdate: acquireUpdate,
@@ -123,18 +130,19 @@ func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excluded
selected := make([]updater.ImageUpdateRecord, 0, len(records))
var targets []arcaneupdater.FrozenUpdateTarget
if len(records) > 0 {
- dockerClient, clientErr := s.dockerClient(ctx)
- if clientErr != nil {
- return ctx, clientErr
- }
- listed, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
+ listed, listErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: false})
+ })
if listErr != nil {
return ctx, listErr
}
// Resolve each candidate's configured image once, not per record.
candidateRefs := make(map[string]string, len(listed.Items))
for _, candidate := range listed.Items {
- candidateRefs[candidate.ID] = refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, candidate))
+ configured, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
+ return imageref.ConfiguredImage(ctx, dockerClient, candidate), nil
+ })
+ candidateRefs[candidate.ID] = refs.NormalizeImageUpdateRef(configured)
}
for _, record := range records {
for _, candidate := range listed.Items {
@@ -205,11 +213,7 @@ func (s *Service) freezeRecordTarget(ctx context.Context, record updater.ImageUp
if pinned := kit.Ternary(record.IsTagUpdate(), "", kit.FromPtr(record.LatestDigest)); pinned != "" {
return pinned, nil
}
- resolver := s.digestResolver()
- if resolver == nil {
- return "", errors.New("cannot freeze update without registry digest resolver")
- }
- return resolver.ImageDigest(ctx, record.NewImageRef())
+ return s.images.RemoteDigest(ctx, record.NewImageRef(), nil)
}()
if err != nil {
return nil, record, err
@@ -247,11 +251,9 @@ func (s *Service) freezeRecordTarget(ctx context.Context, record updater.ImageUp
// FreezeContainer captures the identity evidence of one container.
func (s *Service) FreezeContainer(ctx context.Context, id string) (*arcaneupdater.FrozenUpdateTarget, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return nil, err
- }
- result, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, id, client.ContainerInspectOptions{})
+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, id, client.ContainerInspectOptions{})
+ })
if err != nil {
return nil, err
}
@@ -271,11 +273,7 @@ func (s *Service) FreezeContainer(ctx context.Context, id string) (*arcaneupdate
func (s *Service) PreparePull(ctx context.Context, imageRef string) (string, error) {
if single, ok := ctx.Value(frozenSingleKey{}).(*frozenSingle); ok {
if single.Target.DesiredImageRef == "" {
- resolver := s.digestResolver()
- if resolver == nil {
- return "", errors.New("cannot freeze update without registry digest resolver")
- }
- digest, err := resolver.ImageDigest(ctx, imageRef)
+ digest, err := s.images.RemoteDigest(ctx, imageRef, nil)
if err != nil {
return "", err
}
@@ -334,11 +332,9 @@ func (s *Service) PreparePull(ctx context.Context, imageRef string) (string, err
// ConfirmTarget returns whether the desired effect is confirmed,
// or whether the exact original container is unchanged and safe to resume.
func (s *Service) ConfirmTarget(ctx context.Context, target arcaneupdater.FrozenUpdateTarget) (bool, bool, error) {
- dockerClient, err := s.dockerClient(ctx)
- if err != nil {
- return false, false, err
- }
- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ listed, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if err != nil {
return false, false, err
}
@@ -347,17 +343,21 @@ func (s *Service) ConfirmTarget(ctx context.Context, target arcaneupdater.Frozen
if imageErr != nil {
return ""
}
- image, imageInspectErr := dockerClient.ImageInspect(ctx, immutable)
+ inspected, imageInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, immutable)
+ })
if imageInspectErr != nil {
return ""
}
- return image.ID
+ return inspected.ID
}()
for _, candidate := range listed.Items {
if !slices.ContainsFunc(candidate.Names, func(name string) bool { return strings.TrimPrefix(name, "/") == target.ContainerName }) {
continue
}
- result, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, candidate.ID, client.ContainerInspectOptions{})
+ result, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, candidate.ID, client.ContainerInspectOptions{})
+ })
if inspectErr != nil {
return false, false, inspectErr
}
diff --git a/backend/internal/updater/children/recovery/service_test.go b/backend/internal/updater/children/recovery/service_test.go
index d76390adf..b4fc46b65 100644
--- a/backend/internal/updater/children/recovery/service_test.go
+++ b/backend/internal/updater/children/recovery/service_test.go
@@ -3,11 +3,13 @@ package recovery
import (
"context"
"encoding/json/v2"
+ "errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
+ "github.com/getarcaneapp/arcane/types/v2/containerregistry"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"
"github.com/moby/moby/api/types/container"
@@ -17,9 +19,17 @@ import (
"github.com/stretchr/testify/require"
"go.getarcane.app/updater"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
)
+// unresolvedDigests fails every registry digest lookup.
+type unresolvedDigests struct{}
+
+func (unresolvedDigests) RemoteDigest(context.Context, string, []containerregistry.Credential) (string, error) {
+ return "", errors.New("registry digest unavailable")
+}
+
var neverDeferred = func(updater.ImageUpdateRecord, container.Summary) bool { return false }
func newTestService(t *testing.T, server *httptest.Server, pending func(context.Context) ([]updater.ImageUpdateRecord, error)) *Service {
@@ -28,8 +38,8 @@ func newTestService(t *testing.T, server *httptest.Server, pending func(context.
require.NoError(t, err)
t.Cleanup(func() { _ = dockerClient.Close() })
return NewService(
- func(context.Context) (*client.Client, error) { return dockerClient, nil },
- func() updater.RegistryDigestResolver { return nil },
+ docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),
+ unresolvedDigests{},
pending,
func(context.Context) string { return "" },
func(ctx context.Context) (context.Context, func(), error) { return ctx, func() {}, nil },
diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.go
index 15bade45a..0ced7dc1b 100644
--- a/backend/internal/updater/service.go
+++ b/backend/internal/updater/service.go
@@ -160,7 +160,6 @@ func NewUpdaterService(
upgrade selfUpgradeService,
activityService *activity.ActivityService,
cfg *config.Config,
- _ *runs.Coordinator,
admission *runs.Admission,
roles *role.RoleService,
) (*UpdaterService, error) {
@@ -183,8 +182,8 @@ func NewUpdaterService(
}
// keep: recovery callbacks bind to the allocated service
service.recovery = recovery.NewService(
- service.DockerClient,
- func() updater.RegistryDigestResolver { return registries },
+ localDocker,
+ imageSvc,
service.PendingImageUpdates,
activityIDFromContext,
service.acquireUpdate,
@@ -396,12 +395,11 @@ func (s *UpdaterService) applyScopedUpdates(ctx context.Context, options arcaneu
return docker.ComposeServiceLabel(summary.Labels) != "" && wanted[strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))]
}
if scope == "image" {
- dockerClient, clientErr := s.DockerClient(ctx)
- if clientErr != nil {
- return nil, clientErr
- }
matches = func(summary container.Summary) bool {
- return wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, summary))]
+ configured, _ := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
+ return imageref.ConfiguredImage(ctx, dockerClient, summary), nil
+ })
+ return wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(configured)]
}
}
var matched []string
@@ -646,12 +644,12 @@ func (s *UpdaterService) recordAutoUpdateEvent(ctx context.Context, severity eve
}
}
-// DockerClient returns Arcane's configured Docker client for the updater engine.
+// DockerClient implements updater.DockerClientProvider for the updater engine.
func (s *UpdaterService) DockerClient(ctx context.Context) (*client.Client, error) {
return s.deps.Docker.GetClient(ctx)
}
-// PullImage pulls an image through Arcane's image service, bounded by the
+// PullImage pulls an image through Arcane's image service, which bounds it by the
// dockerImagePullTimeout setting so a stuck pull cannot hold the auto-update run.
func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progress io.Writer) error {
pulledRef, err := s.recovery.PreparePull(ctx, imageRef)
@@ -661,33 +659,17 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
writer := activitylib.NewWriter(ctx, s.deps.Activity, activityIDFromContext(ctx), progress, "Pulling updated images")
defer activitylib.FlushWriter(writer)
- pullCtx, cancelPull := context.WithTimeout(ctx, timeouts.GetDuration(s.deps.Settings.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))
- defer cancelPull()
-
- credentials, err := s.deps.Projects.ResolveRegistryCredentials(pullCtx)
+ credentials, err := s.deps.Projects.ResolveRegistryCredentials(ctx)
if err != nil {
return fmt.Errorf("resolve registry credentials: %w", err)
}
- pull := func(ctx context.Context) error {
- return s.deps.ImagePuller.PullImage(ctx, pulledRef, writer, s.deps.SystemUser, credentials)
- }
- // A frozen digest pull is tagged back to the reference the engine asked for.
- if pulledRef != imageRef {
- dockerClient, clientErr := s.DockerClient(pullCtx)
- if clientErr != nil {
- return clientErr
- }
- if pullErr := imageref.PullAsTag(pullCtx, dockerClient, pulledRef, imageRef, pull); pullErr != nil {
- return pullErr
- }
- } else if pullErr := pull(pullCtx); pullErr != nil {
- return pullErr
- }
- // Reconcile by tag on the run context so a late pull still clears its update records (#4306).
- if reconcileErr := s.deps.ImagePuller.ReconcilePulledImageUpdate(ctx, imageRef); reconcileErr != nil {
- slog.WarnContext(ctx, "failed to reconcile pulled image update state", "image", imageRef, "error", reconcileErr)
- }
- return nil
+ return s.deps.ImagePuller.Pull(ctx, image.PullRequest{
+ Ref: pulledRef,
+ Tag: imageRef,
+ Progress: writer,
+ User: s.deps.SystemUser,
+ Credentials: credentials,
+ })
}
// PendingImageUpdates returns pending image update records from Arcane's database.
@@ -721,17 +703,17 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Ima
if scopedCount == 0 {
return nil, nil
}
- dockerClient, clientErr := s.DockerClient(ctx)
- if clientErr != nil {
- return nil, clientErr
- }
- listed, listErr := dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ listed, listErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ })
if listErr != nil {
return nil, listErr
}
matched := make([]eligibleContainer, 0, len(listed.Items))
for _, cnt := range listed.Items {
- configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)
+ configured, _ := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
+ return imageref.ConfiguredImage(ctx, dockerClient, cnt), nil
+ })
resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
if policyErr != nil || resolved.Strategy == "tag" {
continue
@@ -787,21 +769,23 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record upda
}
// A shared record stays pending while another non-tag-policy container still runs an older image.
- dockerClient, err := s.DockerClient(ctx)
- if err != nil {
- return err
- }
- target, err := dockerClient.ImageInspect(ctx, record.NewImageRef())
+ target, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, record.NewImageRef())
+ })
if err != nil {
return err
}
- listed, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ listed, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ })
if err != nil {
return err
}
for _, cnt := range listed.Items {
// Untagged images list by ID, so an orphaned container matches by its configured image.
- configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)
+ configured, _ := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
+ return imageref.ConfiguredImage(ctx, dockerClient, cnt), nil
+ })
resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
if policyErr != nil || resolved.Strategy == "tag" {
continue
@@ -948,11 +932,9 @@ func (s *UpdaterService) singleContainerActivityRequest(ctx context.Context, con
name := func() string {
lookupCtx, cancelLookup := context.WithTimeout(ctx, 2*time.Second)
defer cancelLookup()
- dockerClient, dockerErr := s.DockerClient(lookupCtx)
- if dockerErr != nil {
- return containerID
- }
- inspected, inspectErr := compat.ContainerInspectWithCompatibility(lookupCtx, dockerClient, containerID, client.ContainerInspectOptions{})
+ inspected, inspectErr := s.deps.Docker.Call(lookupCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ })
if inspectErr != nil {
return containerID
}
@@ -1186,12 +1168,10 @@ func (s *UpdaterService) CollectUsedImages(ctx context.Context) (map[string]stru
}
func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, out map[string]struct{}) error {
- dcli, err := s.DockerClient(ctx)
- if err != nil {
- return err
- }
excludedContainers := docker.ExcludedContainerNameSet(s.deps.Settings.GetStringSetting(ctx, "autoUpdateExcludedContainers", ""))
- listResult, err := dcli.ContainerList(ctx, client.ContainerListOptions{})
+ listResult, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ })
if err != nil {
return err
}
@@ -1216,7 +1196,9 @@ func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, ou
}
// The summary only names an image ID, so the image's tags and the configured reference stand in.
- inspectResult, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dcli, summary.ID, client.ContainerInspectOptions{})
+ inspectResult, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, summary.ID, client.ContainerInspectOptions{})
+ })
if inspectErr != nil {
slog.DebugContext(ctx, "collectUsedImagesFromContainers: container inspect failed", "containerId", summary.ID, "error", inspectErr)
continue
@@ -1227,7 +1209,9 @@ func (s *UpdaterService) collectUsedImagesFromContainers(ctx context.Context, ou
}
if _, inspected := inspectedImages[inspectResult.Container.Image]; !inspected {
inspectedImages[inspectResult.Container.Image] = struct{}{}
- if imageInspect, imageErr := dcli.ImageInspect(ctx, inspectResult.Container.Image); imageErr == nil {
+ if imageInspect, imageErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, inspectResult.Container.Image)
+ }); imageErr == nil {
for _, tag := range imageInspect.RepoTags {
if strings.TrimSpace(tag) != "" && tag != "<none>:<none>" {
addNormalizedImageUpdateRef(ctx, out, tag, "normalizedTagsForContainer repo tag", "imageId", inspectResult.Container.Image)
@@ -1250,13 +1234,14 @@ func (s *UpdaterService) collectUsedImagesFromProjects(ctx context.Context, out
activeProjectNames := map[string]struct{}{}
for _, project := range allProjects {
+ name := strings.TrimSpace(project.Name)
if project.IsArchived ||
(project.Status != projectpkg.ProjectStatusRunning && project.Status != projectpkg.ProjectStatusPartiallyRunning) ||
- strings.TrimSpace(project.Name) == "" {
+ name == "" {
continue
}
- activeProjectNames[strings.TrimSpace(project.Name)] = struct{}{}
- if normalized := projects.NormalizeProjectName(strings.TrimSpace(project.Name)); normalized != "" {
+ activeProjectNames[name] = struct{}{}
+ if normalized := projects.NormalizeProjectName(name); normalized != "" {
activeProjectNames[normalized] = struct{}{}
}
}
@@ -1264,11 +1249,9 @@ func (s *UpdaterService) collectUsedImagesFromProjects(ctx context.Context, out
return nil
}
- dockerClient, err := s.deps.Docker.GetClient(ctx)
- if err != nil {
- return err
- }
- composeContainers, err := projects.ListGlobalComposeContainers(ctx, dockerClient, s.deps.Docker.DockerHost())
+ composeContainers, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]container.Summary, error) {
+ return projects.ListGlobalComposeContainers(ctx, dockerClient, s.deps.Docker.DockerHost())
+ })
if err != nil {
return err
}
diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.go
index 134ddd012..717505fa9 100644
--- a/backend/internal/updater/service_test.go
+++ b/backend/internal/updater/service_test.go
@@ -269,7 +269,6 @@ func TestUpdaterService_ApplyPendingNoRecords(t *testing.T) {
nil,
nil,
nil,
- nil,
admission,
nil,
)
@@ -335,7 +334,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("server label triggers upgrade with system user", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -356,7 +355,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("engine target keeps the selected tag and pulls the frozen digest", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
require.NoError(t, svcErr)
frozen := svc.recovery.WithSingleTarget(ctx, &arcaneupdater.FrozenUpdateTarget{
@@ -379,7 +378,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("agent label triggers upgrade", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane-agent", map[string]string{
@@ -392,7 +391,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "demo", map[string]string{
@@ -406,7 +405,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("upgrade errors are wrapped", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{triggerError: errors.New("upgrade failed")}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -419,7 +418,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
}
func TestUpdaterService_StatusTracking(t *testing.T) {
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
stopContainer := svc.BeginContainerUpdate("container-1")
@@ -445,7 +444,7 @@ func TestUpdaterService_DockerClientAdapter(t *testing.T) {
ctx := t.Context()
t.Run("active engine updates correlate before the stop event is recorded", func(t *testing.T) {
events := event.NewEventService(nil, nil, nil)
- svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(nil, nil, nil, nil, nil, nil, events, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
end := svc.engine.BeginContainerUpdate("updating-container")
require.True(t, events.ShouldSuppressDaemonEvent("container", "updating-container", "", ""))
@@ -459,7 +458,7 @@ func TestUpdaterService_DockerClientAdapter(t *testing.T) {
server := newImagePullServerWithObserver(t, nil, nil)
wantClient := newTestDockerClient(t, server)
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(wantClient)
- svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
gotClient, err := svc.DockerClient(ctx)
@@ -480,10 +479,19 @@ func TestUpdaterService_PullImageAdapter(t *testing.T) {
t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })
server := newImagePullServerWithObserver(t, nil, nil)
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
- imageSvc := image.NewImageService(db, dockerSvc, nil, nil, nil, event.NewEventService(db, nil, nil), nil, nil)
+ imageSvc := image.NewImageService(
+ db,
+ dockerSvc,
+ nil,
+ imageupdate.NewImageUpdateService(db, settingsService, nil, dockerSvc, nil, nil, nil),
+ nil,
+ event.NewEventService(db, nil, nil),
+ settingsService,
+ nil,
+ )
projectSvc := project.NewProjectService(db, nil, nil, nil, testDockerService, nil, nil, nil, nil, nil, nil)
projectSvc.RegistryCredentialsProvider = environment.NewEnvironmentService(db, nil, nil, nil, nil, nil).GetEnabledRegistryCredentials
- svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
var progress bytes.Buffer
@@ -509,16 +517,25 @@ func TestUpdaterService_PullImageAdapter(t *testing.T) {
gotAuth = authHeader
}
})
+ settingsService, err := settings.NewSettingsService(ctx, db)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
registrySvc := registry.NewContainerRegistryService(db, nil, kv.NewKVService(db), nil)
- imageSvc := image.NewImageService(db, dockerSvc, registrySvc, nil, nil, event.NewEventService(db, nil, nil), nil, nil)
+ imageSvc := image.NewImageService(
+ db,
+ dockerSvc,
+ registrySvc,
+ imageupdate.NewImageUpdateService(db, nil, registrySvc, dockerSvc, nil, nil, nil),
+ nil,
+ event.NewEventService(db, nil, nil),
+ settingsService,
+ nil,
+ )
envSvc := environment.NewEnvironmentService(db, nil, nil, nil, nil, nil)
projectSvc := project.NewProjectService(db, nil, nil, nil, testDockerService, nil, nil, nil, nil, nil, nil)
projectSvc.RegistryCredentialsProvider = envSvc.GetEnabledRegistryCredentials
- settingsService, err := settings.NewSettingsService(ctx, db)
- require.NoError(t, err)
- t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })
- svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, settingsService, dockerSvc, projectSvc, nil, nil, nil, imageSvc, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
var progress bytes.Buffer
@@ -565,7 +582,7 @@ func TestUpdaterService_PendingImageUpdatesAdapter(t *testing.T) {
require.NoError(t, db.AutoMigrate(&notification.NotificationSettings{}))
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
records, err := svc.PendingImageUpdates(ctx)
@@ -610,8 +627,12 @@ func TestUpdaterService_PendingImageUpdatesAbortsWhenNotificationEligibilityUnre
db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&imageupdate.ImageUpdateRecord{}, &notification.NotificationSettings{}))
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
- imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, notif, nil)
- svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ }))
+ t.Cleanup(server.Close)
+ imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server)), nil, notif, nil)
+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)
require.NoError(t, err)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
ID: "sha256:pending",
@@ -655,7 +676,7 @@ func TestUpdaterService_PendingImageUpdatesFlushesPendingNotifications(t *testin
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
@@ -686,7 +707,7 @@ func TestUpdaterService_PendingImageUpdatesNoProvidersLeavesUnnotified(t *testin
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, notif, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
require.NoError(t, db.Create(&imageupdate.ImageUpdateRecord{
@@ -710,7 +731,7 @@ func TestUpdaterService_RecordUpdateRunAdapter(t *testing.T) {
ctx := t.Context()
db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}))
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, svcErr)
err := svc.RecordUpdateRun(ctx, updater.ResourceResult{
@@ -756,7 +777,7 @@ func TestUpdaterService_RecordUpdateRunAppendsActivityMessage(t *testing.T) {
db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&AutoUpdateRecord{}, &activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
- svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil)
require.NoError(t, svcErr)
for _, tt := range []struct {
@@ -841,7 +862,7 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivity(t *testing
harness := flowtest.New(t, activityService)
admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())
require.NoError(t, admission.Register(harness.Runtime))
- svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, admission, nil)
+ svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, admission, nil)
require.NoError(t, err)
require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))
harness.Start(t)
@@ -894,7 +915,7 @@ func TestUpdaterService_AcceptSingleContainerUpdateFailureFinalizes(t *testing.T
db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
- svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil)
require.NoError(t, err)
// A host that has not started cannot start the workflow.
require.NoError(t, svc.execution.RegisterWorkflows(flowtest.New(t, activityService).Engine))
@@ -982,7 +1003,7 @@ func TestUpdaterService_SingleContainerActivityResultMessages(t *testing.T) {
db := setupProjectTestDB(t)
require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))
activityService := activity.NewActivityService(db, nil)
- svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, activityService, nil, nil, nil)
require.NoError(t, err)
started, err := activityService.StartActivity(
t.Context(),
@@ -1128,7 +1149,6 @@ func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjects(t *
nil,
nil,
nil,
- nil,
admission,
nil,
)
@@ -1282,7 +1302,6 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgrade(
mockUpgrade,
nil,
nil,
- nil,
admission,
nil,
)
@@ -1555,7 +1574,7 @@ func TestUpdaterService_ScopedPendingRecordsIsolation(t *testing.T) {
require.NoError(t, db.AutoMigrate(&notification.NotificationSettings{}))
notif := notification.NewNotificationService(db, nil, nil, nil, nil)
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, dockerSvc, nil, notif, nil)
- svc, err := NewUpdaterService(db, nil, dockerSvc, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, imageUpdates, nil, nil, nil, notif, nil, nil, nil, nil, nil)
require.NoError(t, err)
records, err := svc.PendingImageUpdates(t.Context())
require.NoError(t, err)
@@ -1632,7 +1651,7 @@ func TestUpdaterService_SharedDigestRecordClearing(t *testing.T) {
}
server := newUpdaterApplyPendingDockerServer(t, containers, nil, nil, map[string]dockertypesimage.InspectResponse{imageRef: {ID: "new-image"}})
dockerSvc := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
- svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, dockerSvc, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
require.NoError(t, svc.ClearImageUpdateRecord(t.Context(), updater.ImageUpdateRecord{
ContainerID: "updated",
@@ -1783,7 +1802,6 @@ func TestSingleUpdatePersistsSuccessfulActivity(t *testing.T) {
nil,
activityService,
nil,
- nil,
admission,
nil,
)
@@ -1806,7 +1824,7 @@ func TestUpdaterService_WorkflowDefinitionsAreVersioned(t *testing.T) {
harness := flowtest.New(t, activity.NewActivityService(db, nil))
imageUpdates := imageupdate.NewImageUpdateService(db, nil, nil, nil, nil, nil, nil)
require.NoError(t, imageUpdates.RegisterWorkflows(harness.Engine))
- svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, nil, nil, nil, imageUpdates, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
require.NoError(t, svc.RegisterWorkflows(harness.Engine))
harness.Start(t)
@@ -1845,7 +1863,7 @@ func TestUpdaterService_AutoUpdateDeferral(t *testing.T) {
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(ctx))) })
require.NoError(t, settingsService.SetIntSetting(ctx, "autoUpdateDelay", 24))
- svc, err := NewUpdaterService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
+ svc, err := NewUpdaterService(db, settingsService, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
require.NoError(t, err)
deferred, err := svc.autoUpdateDeferral(ctx, now)
diff --git a/backend/internal/version/service.go b/backend/internal/version/service.go
index c97f90b04..7e74ebcc7 100644
--- a/backend/internal/version/service.go
+++ b/backend/internal/version/service.go
@@ -12,7 +12,6 @@ import (
"github.com/distribution/reference"
"github.com/getarcaneapp/arcane/types/v2/version"
- containertypes "github.com/moby/moby/api/types/container"
"github.com/moby/moby/client"
"github.com/samber/hot"
"github.com/samber/mo"
@@ -84,7 +83,7 @@ func NewVersionService(
loader := func(_ []struct{}) (map[struct{}]latestRelease, error) {
ctx, cancel := context.WithTimeout(context.Background(), defaultRequestTimeout) //nolint:forbidigo // Cache revalidation runs independently of request cancellation.
defer cancel()
- release, err := service.fetchLatestReleaseInternal(ctx)
+ release, err := service.fetchLatestRelease(ctx)
if err != nil {
return nil, err
}
@@ -99,7 +98,7 @@ func NewVersionService(
return service
}
-func (s *VersionService) getLatestReleaseInternal(_ context.Context) (latestRelease, error) {
+func (s *VersionService) getLatestRelease(_ context.Context) (latestRelease, error) {
release, found, err := s.cache.Get(struct{}{})
if err != nil {
return latestRelease{}, err
@@ -110,7 +109,7 @@ func (s *VersionService) getLatestReleaseInternal(_ context.Context) (latestRele
return release, nil
}
-func (s *VersionService) fetchLatestReleaseInternal(ctx context.Context) (latestRelease, error) {
+func (s *VersionService) fetchLatestRelease(ctx context.Context) (latestRelease, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, versionCheckURL, http.NoBody)
if err != nil {
return latestRelease{}, fmt.Errorf("create GitHub request: %w", err)
@@ -146,7 +145,7 @@ func (s *VersionService) fetchLatestReleaseInternal(ctx context.Context) (latest
}
func (s *VersionService) GetLatestVersion(ctx context.Context) (string, error) {
- rel, err := s.getLatestReleaseInternal(ctx)
+ rel, err := s.getLatestRelease(ctx)
if err != nil {
return "", err
}
@@ -203,57 +202,41 @@ func (s *VersionService) GetVersionInformation(ctx context.Context, currentVersi
}
cur := s.normalizeVersion(currentVersion)
- check := &version.Check{
- CurrentVersion: cur,
- ReleaseURL: s.ReleaseURL(""),
- UpdateAvailable: false,
- }
-
- if s.disabled {
- return check, nil
- }
-
- latest, err := s.GetLatestVersion(ctx)
- if err != nil {
- return check, err
- }
-
+ latest, err := func() (string, error) {
+ if s.disabled {
+ return "", nil
+ }
+ return s.GetLatestVersion(ctx)
+ }()
// Skip when latest is semver-older than current (e.g. a prerelease build
// asking the stable release feed) — a downgrade is not a newest version.
- if latest != "" && !s.IsNewer(cur, latest) {
- check.NewestVersion = latest
- check.UpdateAvailable = s.IsNewer(latest, cur)
- check.ReleaseURL = s.ReleaseURL(latest)
- }
-
- return check, nil
+ newest := kit.Ternary(err == nil && latest != "" && !s.IsNewer(cur, latest), latest, "")
+ return &version.Check{
+ CurrentVersion: cur,
+ NewestVersion: newest,
+ UpdateAvailable: newest != "" && s.IsNewer(newest, cur),
+ ReleaseURL: s.ReleaseURL(newest),
+ }, err
}
-// isNextBuildInternal reports whether this build tracks the next channel: a
+// isNextBuild reports whether this build tracks the next channel: a
// -next. prerelease version, or a container running the rolling next image tag.
-func (s *VersionService) isNextBuildInternal(currentTag string) bool {
+func (s *VersionService) isNextBuild(currentTag string) bool {
return strings.Contains(strings.ToLower(strings.TrimSpace(s.version)), "-next.") ||
strings.TrimSpace(currentTag) == "next"
}
-// resolveNextVersionInternal returns the normalized version label of the image
+// resolveNextVersion returns the normalized version label of the image
// an upgrade would pull, or "" when it cannot be resolved. It prefers the
// digest reference so the reported version and NewestDigest describe the same
// artifact.
-func (s *VersionService) resolveNextVersionInternal(ctx context.Context, imageRef, tag, newestDigest string) string {
- if s.containerRegistryService == nil || strings.TrimSpace(imageRef) == "" {
+func (s *VersionService) resolveNextVersion(ctx context.Context, imageRef, tag, newestDigest string) string {
+ newestDigest, tag = strings.TrimSpace(newestDigest), strings.TrimSpace(tag)
+ if s.containerRegistryService == nil || strings.TrimSpace(imageRef) == "" || (newestDigest == "" && tag == "") {
return ""
}
- var lookupRef string
- switch {
- case strings.TrimSpace(newestDigest) != "":
- lookupRef = imageRef + "@" + strings.TrimSpace(newestDigest)
- case strings.TrimSpace(tag) != "":
- lookupRef = imageRef + ":" + strings.TrimSpace(tag)
- default:
- return ""
- }
+ lookupRef := kit.Ternary(newestDigest != "", imageRef+"@"+newestDigest, imageRef+":"+tag)
label, err := s.containerRegistryService.ImageVersionLabel(ctx, lookupRef)
if err != nil {
@@ -269,20 +252,8 @@ func (s *VersionService) resolveNextVersionInternal(ctx context.Context, imageRe
return normalized
}
-// isSemverVersion checks if a version string is semver-based (e.g., v1.0.0)
-func (s *VersionService) isSemverVersion() bool {
- return semver.IsValid(kit.EnsurePrefix(s.version, "v"))
-}
-
-// getDisplayVersion formats the version for display purposes
-// Semver versions (including prereleases like 2.4.0-next.1) display as v<version>
-func (s *VersionService) getDisplayVersion() string {
- versionValue := strings.TrimPrefix(strings.TrimSpace(s.version), "v")
- return kit.Ternary(s.isSemverVersion(), "v"+versionValue, versionValue)
-}
-
-// updateCheckImageRefInternal points the digest lookup at the configured registry; "auto" keeps the running reference.
-func (s *VersionService) updateCheckImageRefInternal(currentImageRef string) string {
+// updateCheckImageRef points the digest lookup at the configured registry; "auto" keeps the running reference.
+func (s *VersionService) updateCheckImageRef(currentImageRef string) string {
if s.settingsService == nil {
return currentImageRef
}
@@ -312,18 +283,66 @@ func (s *VersionService) updateCheckImageRefInternal(currentImageRef string) str
// GetAppVersionInfo returns application version information including display version
func (s *VersionService) GetAppVersionInfo(ctx context.Context) *version.Info {
- isSemver := s.isSemverVersion()
+ isSemver := semver.IsValid(kit.EnsurePrefix(s.version, "v"))
ver := s.normalizeVersion(s.version)
+ // Semver versions (including prereleases like 2.4.0-next.1) display as v<version>.
+ displayVersion := strings.TrimPrefix(strings.TrimSpace(s.version), "v")
// Always detect current image info
currentTag, currentDigest, currentImageRef, currentImageID := s.detectCurrentImageInfo(ctx)
- // Build base info struct (always populated)
- info := &version.Info{
+ // Update fields stay empty when update checks are disabled.
+ update := func() version.Info {
+ if s.disabled {
+ return version.Info{}
+ }
+ checkImageRef := s.updateCheckImageRef(currentImageRef)
+ digestUpdateAvailable, latestDigest := s.storedOrDigestBasedUpdate(ctx, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef)
+
+ switch {
+ case isSemver && s.isNextBuild(currentTag):
+ nextVersion := s.resolveNextVersion(ctx, checkImageRef, currentTag, latestDigest)
+ return version.Info{
+ NewestDigest: latestDigest,
+ NewestVersion: kit.Ternary(nextVersion != "" && semver.Compare(nextVersion, ver) >= 0, nextVersion, ""),
+ UpdateAvailable: digestUpdateAvailable || (nextVersion != "" && s.IsNewer(nextVersion, ver)),
+ }
+ case isSemver:
+ // Never surface a release older than the running version as the target.
+ rel, err := s.getLatestRelease(ctx)
+ if err != nil || rel.TagName == "" || s.IsNewer(ver, rel.TagName) {
+ return version.Info{NewestDigest: latestDigest}
+ }
+ return version.Info{
+ NewestDigest: latestDigest,
+ NewestVersion: rel.TagName,
+ UpdateAvailable: s.IsNewer(rel.TagName, ver),
+ ReleaseURL: s.ReleaseURL(rel.TagName),
+ ReleaseNotes: rel.Body,
+ ReleasedAt: rel.PublishedAt,
+ }
+ }
+
+ // Best-effort: pull release notes for non-semver track too, so the modal can preview
+ // the latest tagged release even when the running build is digest-tracking.
+ rel, err := s.getLatestRelease(ctx)
+ if err != nil || rel.TagName == "" {
+ return version.Info{NewestDigest: latestDigest, UpdateAvailable: digestUpdateAvailable}
+ }
+ return version.Info{
+ NewestDigest: latestDigest,
+ UpdateAvailable: digestUpdateAvailable,
+ ReleaseURL: s.ReleaseURL(rel.TagName),
+ ReleaseNotes: rel.Body,
+ ReleasedAt: rel.PublishedAt,
+ }
+ }()
+
+ return &version.Info{
CurrentVersion: ver,
CurrentTag: currentTag,
CurrentDigest: currentDigest,
- DisplayVersion: s.getDisplayVersion(),
+ DisplayVersion: kit.Ternary(isSemver, "v"+displayVersion, displayVersion),
Revision: s.revision,
ShortRevision: config.ShortRevision(),
GoVersion: config.GoVersion(),
@@ -332,55 +351,17 @@ func (s *VersionService) GetAppVersionInfo(ctx context.Context) *version.Info {
EnabledFeatures: append(kit.Unique(kit.TrimNonEmpty(strings.Split(strings.ToLower(buildables.EnabledFeatures), ","))), apns.FeatureName),
BuildTime: config.BuildTime,
IsSemverVersion: isSemver,
- UpdateAvailable: false,
- }
-
- // If update checks disabled, return base info
- if s.disabled {
- return info
- }
-
- checkImageRef := s.updateCheckImageRefInternal(currentImageRef)
- digestUpdateAvailable, latestDigest := s.storedOrDigestBasedUpdateInternal(ctx, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef)
- if latestDigest != "" {
- info.NewestDigest = latestDigest
- }
-
- switch {
- case isSemver && s.isNextBuildInternal(currentTag):
- nextVersion := s.resolveNextVersionInternal(ctx, checkImageRef, currentTag, latestDigest)
- if nextVersion != "" && semver.Compare(nextVersion, ver) >= 0 {
- info.NewestVersion = nextVersion
- }
- info.UpdateAvailable = digestUpdateAvailable || (nextVersion != "" && s.IsNewer(nextVersion, ver))
- case isSemver:
- // Never surface a release older than the running version as the target.
- rel, err := s.getLatestReleaseInternal(ctx)
- if err == nil && rel.TagName != "" && !s.IsNewer(ver, rel.TagName) {
- info.NewestVersion = rel.TagName
- info.UpdateAvailable = s.IsNewer(rel.TagName, ver)
- info.ReleaseURL = s.ReleaseURL(rel.TagName)
- info.ReleaseNotes = rel.Body
- info.ReleasedAt = rel.PublishedAt
- }
- default:
- // Best-effort: pull release notes for non-semver track too, so the modal can preview
- // the latest tagged release even when the running build is digest-tracking.
- if rel, err := s.getLatestReleaseInternal(ctx); err == nil && rel.TagName != "" {
- info.ReleaseNotes = rel.Body
- info.ReleasedAt = rel.PublishedAt
- if info.ReleaseURL == "" {
- info.ReleaseURL = s.ReleaseURL(rel.TagName)
- }
- }
- info.UpdateAvailable = digestUpdateAvailable
+ NewestVersion: update.NewestVersion,
+ NewestDigest: update.NewestDigest,
+ UpdateAvailable: update.UpdateAvailable,
+ ReleaseURL: update.ReleaseURL,
+ ReleaseNotes: update.ReleaseNotes,
+ ReleasedAt: update.ReleasedAt,
}
-
- return info
}
-// storedOrDigestBasedUpdateInternal uses the stored poller row only when the check targets the running reference.
-func (s *VersionService) storedOrDigestBasedUpdateInternal(ctx context.Context, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef string) (bool, string) {
+// storedOrDigestBasedUpdate uses the stored poller row only when the check targets the running reference.
+func (s *VersionService) storedOrDigestBasedUpdate(ctx context.Context, currentImageID, currentTag, currentDigest, currentImageRef, checkImageRef string) (bool, string) {
if s.imageUpdateService != nil && strings.TrimSpace(currentImageID) != "" && checkImageRef == currentImageRef {
record, found, err := s.imageUpdateService.StoredUpdateByImageID(ctx, currentImageID)
if err != nil {
@@ -390,11 +371,21 @@ func (s *VersionService) storedOrDigestBasedUpdateInternal(ctx context.Context,
}
}
- if currentTag != "" && currentDigest != "" && checkImageRef != "" && s.containerRegistryService != nil {
- return s.checkDigestBasedUpdate(ctx, currentTag, currentDigest, checkImageRef)
+ if currentTag == "" || currentDigest == "" || checkImageRef == "" || s.containerRegistryService == nil {
+ return false, ""
+ }
+
+ latestDigest, err := s.containerRegistryService.ImageDigest(ctx, checkImageRef+":"+currentTag)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to fetch latest digest for tag", "tag", currentTag, "error", err)
+ return false, ""
}
- return false, ""
+ updateAvailable := currentDigest != latestDigest && latestDigest != ""
+ if updateAvailable {
+ slog.InfoContext(ctx, "Digest-based update available", "tag", currentTag, "currentDigest", currentDigest, "latestDigest", latestDigest)
+ }
+ return updateAvailable, latestDigest
}
// detectCurrentImageInfo attempts to detect the current container's image tag and digest
@@ -410,130 +401,56 @@ func (s *VersionService) detectCurrentImageInfo(ctx context.Context) (tag, diges
return "", "", "", ""
}
- containerId := s.detectContainerID(ctx, dockerClient)
+ containerId, err := cgroup.CurrentContainerID()
+ if err != nil {
+ // Fall back to the Arcane label when cgroup/hostname detection fails.
+ slog.DebugContext(ctx, "detectCurrentImageInfo: cgroup detection failed, trying label fallback", "error", err)
+ containerId = libarcane.FindArcaneContainerIDByLabel(ctx, dockerClient)
+ }
if containerId == "" {
slog.DebugContext(ctx, "detectCurrentImageInfo: could not detect container ID")
return "", "", "", ""
}
slog.DebugContext(ctx, "detectCurrentImageInfo: detected container", "containerId", containerId)
- inspectResult, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, containerId, client.ContainerInspectOptions{})
+ inspectResult, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, cli, containerId, client.ContainerInspectOptions{})
+ })
if err != nil {
slog.DebugContext(ctx, "detectCurrentImageInfo: failed to inspect container", "containerId", containerId, "error", err)
return "", "", "", ""
}
- container := inspectResult.Container
- imageID = container.Image
-
- configImage := ""
- if container.Config != nil {
- configImage = container.Config.Image
- }
-
- // Parse tag from container config image (user-specified reference)
- tag = s.extractTagFromImageRef(configImage)
-
- // Get digest and normalized imageRef from container image
- imageRef, digest = s.extractImageDetails(ctx, dockerClient, container)
-
- // Fallback to container config image if RepoDigests didn't provide imageRef
- if imageRef == "" {
- imageRef = s.normalizeImageRef(configImage)
- }
-
- return tag, digest, imageRef, imageID
-}
+ imageID = inspectResult.Container.Image
+ configImage := kit.FromPtr(inspectResult.Container.Config).Image
-// detectContainerID tries to get the current container ID, falling back to label-based detection
-func (s *VersionService) detectContainerID(ctx context.Context, dockerClient *client.Client) string {
- containerId, err := s.getCurrentContainerID()
- if err == nil {
- slog.DebugContext(ctx, "detectContainerID: found via getCurrentContainerID", "containerId", containerId)
- return containerId
+ // The tag comes from the user-specified config image; the repository falls back to it too.
+ tag, imageRef = "latest", configImage
+ if named, parseErr := reference.ParseNormalizedNamed(configImage); parseErr == nil {
+ imageRef = named.Name()
+ if tagged, ok := named.(reference.Tagged); ok {
+ tag = tagged.Tag()
+ }
}
- slog.DebugContext(ctx, "detectContainerID: getCurrentContainerID failed, trying label fallback", "error", err)
-
- // Fallback: locate the Arcane container by label (works even when cgroup/hostname detection fails)
- return libarcane.FindArcaneContainerIDByLabel(ctx, dockerClient)
-}
-// extractImageDetails extracts digest and imageRef from a container's image
-func (s *VersionService) extractImageDetails(ctx context.Context, dockerClient *client.Client, container containertypes.InspectResponse) (imageRef, digest string) {
- if container.Image == "" {
- return "", ""
+ if imageID == "" {
+ return tag, "", imageRef, imageID
}
-
- imageInspect, err := dockerClient.ImageInspect(ctx, container.Image)
+ imageInspect, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {
+ return cli.ImageInspect(ctx, imageID)
+ })
if err != nil {
- return "", ""
+ return tag, "", imageRef, imageID
}
-
- // Extract digest and repository from first RepoDigest using reference library
for _, repoDigest := range imageInspect.RepoDigests {
- named, parseNormalizedNamedErr := reference.ParseNormalizedNamed(repoDigest)
- if parseNormalizedNamedErr != nil {
+ named, parseErr := reference.ParseNormalizedNamed(repoDigest)
+ if parseErr != nil {
continue
}
if digested, ok := named.(reference.Digested); ok {
- return named.Name(), string(digested.Digest())
+ return tag, string(digested.Digest()), named.Name(), imageID
}
}
-
- return "", ""
-}
-
-// normalizeImageRef extracts just the repository name from an image reference
-func (s *VersionService) normalizeImageRef(configImage string) string {
- if named, err := reference.ParseNormalizedNamed(configImage); err == nil {
- return named.Name()
- }
- return configImage
-}
-
-// getCurrentContainerID detects if we're running in Docker via cgroup, mountinfo, or hostname
-func (s *VersionService) getCurrentContainerID() (string, error) {
- return cgroup.CurrentContainerID()
-}
-
-// extractTagFromImageRef extracts the tag from an image reference using distribution/reference
-func (s *VersionService) extractTagFromImageRef(imageRef string) string {
- named, err := reference.ParseNormalizedNamed(imageRef)
- if err != nil {
- return "latest"
- }
-
- tagged, ok := named.(reference.Tagged)
- if ok {
- return tagged.Tag()
- }
-
- return "latest"
-}
-
-// checkDigestBasedUpdate checks if there's a newer digest for the current tag
-func (s *VersionService) checkDigestBasedUpdate(ctx context.Context, currentTag, currentDigest, currentImageRef string) (updateAvailable bool, latestDigest string) {
- if currentTag == "" || currentDigest == "" || currentImageRef == "" {
- return false, ""
- }
-
- // Build full image reference with tag
- imageRef := fmt.Sprintf("%s:%s", currentImageRef, currentTag)
-
- // Fetch latest digest from registry
- latestDigest, err := s.containerRegistryService.ImageDigest(ctx, imageRef)
- if err != nil {
- slog.WarnContext(ctx, "Failed to fetch latest digest for tag", "tag", currentTag, "error", err)
- return false, ""
- }
-
- // Compare digests - if they differ, an update is available
- updateAvailable = currentDigest != latestDigest && latestDigest != ""
-
- if updateAvailable {
- slog.InfoContext(ctx, "Digest-based update available", "tag", currentTag, "currentDigest", currentDigest, "latestDigest", latestDigest)
- }
-
- return updateAvailable, latestDigest
+ return tag, "", imageRef, imageID
}
// RunStreamProducer shares one version poll loop across every connected client and emits a snapshot whenever it changes.
diff --git a/backend/internal/version/service_test.go b/backend/internal/version/service_test.go
index e3644ee18..4e092dd6e 100644
--- a/backend/internal/version/service_test.go
+++ b/backend/internal/version/service_test.go
@@ -35,7 +35,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
)
-func TestVersionService_GetAppVersionInfoDoesNotUseStoredDigestUpdateForSemverBuildInternal(t *testing.T) {
+func TestVersionService_GetAppVersionInfoDoesNotUseStoredDigestUpdateForSemverBuild(t *testing.T) {
ctx := t.Context()
db := setupImageUpdateTestDB(t)
@@ -117,7 +117,7 @@ func TestVersionService_GetAppVersionInfoDoesNotUseStoredDigestUpdateForSemverBu
Request: req,
}, nil
})}
- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
svc := NewVersionService(httpClient, false, "1.2.3", "revision", nil, dockerService, imageUpdateService, nil)
@@ -140,7 +140,7 @@ func TestVersionService_GetAppVersionInfoDisplaysSemverNextVersion(t *testing.T)
assert.True(t, info.IsSemverVersion)
}
-func TestVersionService_GetAppVersionInfoPreservesDevVersionInternal(t *testing.T) {
+func TestVersionService_GetAppVersionInfoPreservesDevVersion(t *testing.T) {
svc := NewVersionService(nil, true, "dev", "unknown", nil, nil, nil, nil)
info := svc.GetAppVersionInfo(t.Context())
@@ -169,8 +169,8 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
-// newTestDockerClientInternal builds a Docker client pointed at a fake Docker HTTP server.
-func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
+// newTestDockerClient builds a Docker client pointed at a fake Docker HTTP server.
+func newTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
httpClient := server.Client()
@@ -188,7 +188,7 @@ func newTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.
return cli
}
-func TestVersionService_IsNextBuildInternal(t *testing.T) {
+func TestVersionService_IsNextBuild(t *testing.T) {
tests := []struct {
name string
version string
@@ -203,14 +203,14 @@ func TestVersionService_IsNextBuildInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
svc := NewVersionService(nil, true, tt.version, "rev", nil, nil, nil, nil)
- assert.Equal(t, tt.want, svc.isNextBuildInternal(tt.currentTag))
+ assert.Equal(t, tt.want, svc.isNextBuild(tt.currentTag))
})
}
}
-// newLabelRegistryTestServerInternal starts an in-memory OCI registry and
+// newLabelRegistryTestServer starts an in-memory OCI registry and
// returns its host as "localhost:PORT" so go-containerregistry uses plain HTTP.
-func newLabelRegistryTestServerInternal(t *testing.T) string {
+func newLabelRegistryTestServer(t *testing.T) string {
t.Helper()
server := httptest.NewServer(ggcrregistry.New(ggcrregistry.Logger(log.New(io.Discard, "", 0))))
t.Cleanup(server.Close)
@@ -219,9 +219,9 @@ func newLabelRegistryTestServerInternal(t *testing.T) string {
return "localhost:" + serverURL.Port()
}
-// newNextChannelDockerServerInternal fakes a Docker daemon whose Arcane
+// newNextChannelDockerServer fakes a Docker daemon whose Arcane
// container runs repo:tag with the given current repo digest.
-func newNextChannelDockerServerInternal(t *testing.T, containerID, imageID, repo, tag, currentDigest string) *httptest.Server {
+func newNextChannelDockerServer(t *testing.T, containerID, imageID, repo, tag, currentDigest string) *httptest.Server {
t.Helper()
imageRef := repo + ":" + tag
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -256,7 +256,7 @@ func newNextChannelDockerServerInternal(t *testing.T, containerID, imageID, repo
return server
}
-func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *testing.T) {
+func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabel(t *testing.T) {
ctx := t.Context()
db := setupImageUpdateTestDB(t)
@@ -264,7 +264,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *te
containerID = "arcane-container-1234567890"
imageID = "sha256:arcane-image"
)
- registryHost := newLabelRegistryTestServerInternal(t)
+ registryHost := newLabelRegistryTestServer(t)
repo := registryHost + "/getarcaneapp/arcane"
img, err := random.Image(64, 1)
@@ -282,7 +282,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *te
require.NoError(t, err)
currentDigest := digest.FromString("current-arcane").String()
- server := newNextChannelDockerServerInternal(t, containerID, imageID, repo, "next", currentDigest)
+ server := newNextChannelDockerServer(t, containerID, imageID, repo, "next", currentDigest)
require.NoError(t, db.WithContext(ctx).Create(&imageupdate.ImageUpdateRecord{
ID: imageID,
@@ -306,7 +306,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *te
Request: req,
}, nil
})}
- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
registrySvc := registry.NewContainerRegistryService(nil, nil, nil, nil)
svc := NewVersionService(httpClient, false, "2.8.0-next.65", "revision", registrySvc, dockerService, imageUpdateService, nil)
@@ -322,7 +322,7 @@ func TestVersionService_GetAppVersionInfoNextChannelUsesImageLabelInternal(t *te
assert.False(t, githubHit, "next channel must not consult the GitHub stable release feed")
}
-func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigestInternal(t *testing.T) {
+func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDigest(t *testing.T) {
ctx := t.Context()
db := setupImageUpdateTestDB(t)
@@ -331,12 +331,12 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDiges
imageID = "sha256:arcane-image"
)
// Registry with no image pushed: the label fetch fails.
- registryHost := newLabelRegistryTestServerInternal(t)
+ registryHost := newLabelRegistryTestServer(t)
repo := registryHost + "/getarcaneapp/arcane"
currentDigest := digest.FromString("current-arcane").String()
latestDigest := digest.FromString("latest-arcane").String()
- server := newNextChannelDockerServerInternal(t, containerID, imageID, repo, "next", currentDigest)
+ server := newNextChannelDockerServer(t, containerID, imageID, repo, "next", currentDigest)
require.NoError(t, db.WithContext(ctx).Create(&imageupdate.ImageUpdateRecord{
ID: imageID,
@@ -350,7 +350,7 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDiges
CheckTime: time.Now().UTC(),
}).Error)
- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClientInternal(t, server))
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newTestDockerClient(t, server))
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
registrySvc := registry.NewContainerRegistryService(nil, nil, nil, nil)
svc := NewVersionService(nil, false, "2.8.0-next.65", "revision", registrySvc, dockerService, imageUpdateService, nil)
@@ -363,7 +363,7 @@ func TestVersionService_GetAppVersionInfoNextChannelLabelFailureFallsBackToDiges
assert.Equal(t, latestDigest, info.NewestDigest)
}
-func TestVersionService_GetAppVersionInfoSuppressesOlderStableForPrereleaseInternal(t *testing.T) {
+func TestVersionService_GetAppVersionInfoSuppressesOlderStableForPrerelease(t *testing.T) {
httpClient := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
diff --git a/backend/internal/volume/children/backup/children/restore/service.go b/backend/internal/volume/children/backup/children/restore/service.go
index 923cc863e..f61e9f8ae 100644
--- a/backend/internal/volume/children/backup/children/restore/service.go
+++ b/backend/internal/volume/children/backup/children/restore/service.go
@@ -20,6 +20,7 @@ import (
"github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/client"
+ "go.getarcane.app/kit/pkg"
"github.com/getarcaneapp/arcane/backend/v2/internal/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -31,7 +32,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/backupbrowser"
)
-const restoreBackupFilesScriptInternal = `set -e
+const restoreBackupFilesScript = `set -e
archive="$1"
shift
archive_mode=$(stat -c '%A' -- "$archive" 2>/dev/null) || { echo ARCANE_NOT_FOUND >&2; exit 44; }
@@ -50,12 +51,12 @@ type Dependencies struct {
Run func(ctx context.Context, backupID string) (*volume.Backup, error)
Repository func(ctx context.Context, dockerClient *client.Client, entry *volume.Backup) (backup.Repository, string, error)
Password func(ctx context.Context, dockerClient *client.Client, repositories ...backup.Repository) (string, error)
- StopContainers func(ctx context.Context, dockerClient *client.Client, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error)
- StartContainers func(ctx context.Context, dockerClient *client.Client, stopped []container.Summary, user usertypes.Actor) ([]container.Summary, error)
+ StopContainers func(ctx context.Context, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error)
+ StartContainers func(ctx context.Context, stopped []container.Summary, user usertypes.Actor) ([]container.Summary, error)
SafetyBackup func(ctx context.Context, volumeName string, user usertypes.Actor) (string, error)
ArchivePaths func(ctx context.Context, backupID string) ([]string, error)
ArchiveFilename func(backupID string) (string, error)
- HelperImage func(ctx context.Context, dockerClient *client.Client) (string, error)
+ HelperImage func(ctx context.Context) (string, error)
StorageMount func(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error)
AcquireHelper func(ctx context.Context, volumeName string) (string, func(), error)
Exec func(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error)
@@ -96,11 +97,11 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string
return err
}
}
- stopped, err := s.deps.StopContainers(ctx, dockerClient, volumeName, user, true)
+ stopped, err := s.deps.StopContainers(ctx, volumeName, user, true)
containersStopped := len(stopped) > 0
defer func() {
if containersStopped {
- _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)
err = errors.Join(err, restartErr)
}
}()
@@ -110,11 +111,15 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string
// A discovered backup may target a volume missing on this instance; create it and skip the safety backup of an empty volume.
preBackupID := ""
var createdVolume bool
- if _, inspectErr := dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{}); inspectErr != nil {
+ if _, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{})
+ }); inspectErr != nil {
if !errdefs.IsNotFound(inspectErr) {
return fmt.Errorf("failed to inspect volume for restore: %w", inspectErr)
}
- if _, createErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: volumeName}); createErr != nil {
+ if _, createErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {
+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: volumeName})
+ }); createErr != nil {
return fmt.Errorf("failed to create missing volume for restore: %w", createErr)
}
createdVolume = true
@@ -126,7 +131,7 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string
}
}
if entry.Format == volume.BackupFormatArchive {
- if restoreArchiveBackupErr := s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID); restoreArchiveBackupErr != nil {
+ if restoreArchiveBackupErr := s.restoreArchiveBackup(ctx, dockerClient, volumeName, backupID, nil); restoreArchiveBackupErr != nil {
return restoreArchiveBackupErr
}
} else if restoreSnapshotErr := s.deps.Engine.RestoreSnapshot(
@@ -147,7 +152,7 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string
return fmt.Errorf("failed to restore Rustic snapshot: %w", restoreSnapshotErr)
}
if containersStopped {
- stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)
containersStopped = len(stopped) > 0
if err != nil {
return err
@@ -163,57 +168,50 @@ func (s *Service) RestoreBackup(ctx context.Context, volumeName, backupID string
return nil
}
-type selectionInternal struct {
+type selectionPlan struct {
entries []backuptypes.BackupFileEntry
archivePaths []string
globalRoot bool
}
-func (
- s *Service,
-) resolveRestoreSelectionInternal(
+func (s *Service) resolveRestoreSelection(
ctx context.Context,
dockerClient *client.Client,
entry *volume.Backup,
repository backup.Repository,
snapshotID string,
selection backuptypes.RestoreSelection,
-) (
- selectionInternal,
- error,
-) {
+) (selectionPlan, error) {
if selection.SelectAll && strings.TrimSpace(selection.Search) == "" {
if _, err := backupbrowser.NormalizeSelection(selection, []backuptypes.BackupFileEntry{{Path: "", IsDirectory: true}}); err != nil {
- return selectionInternal{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)
+ return selectionPlan{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)
}
- return selectionInternal{globalRoot: true}, nil
+ return selectionPlan{globalRoot: true}, nil
}
- resolved := selectionInternal{}
- var eligible []backuptypes.BackupFileEntry
- var err error
- if entry.Format == volume.BackupFormatArchive {
- resolved.archivePaths, err = s.deps.ArchivePaths(ctx, entry.ID)
- eligible = backupbrowser.BuildEntries(resolved.archivePaths, "", true)
- } else {
- var listed []string
+ paths, err := func() ([]string, error) {
+ if entry.Format == volume.BackupFormatArchive {
+ return s.deps.ArchivePaths(ctx, entry.ID)
+ }
password, passwordErr := s.deps.Password(ctx, dockerClient, repository)
if passwordErr != nil {
- return selectionInternal{}, passwordErr
+ return nil, passwordErr
}
- listed, err = s.deps.Engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)
- eligible = backupbrowser.BuildEntries(listed, "", true)
- }
+ return s.deps.Engine.ListSnapshotFiles(ctx, dockerClient, repository, password, snapshotID, "", true)
+ }()
if err != nil {
- return selectionInternal{}, err
+ return selectionPlan{}, err
}
- resolved.entries, err = backupbrowser.NormalizeSelection(selection, eligible)
+ entries, err := backupbrowser.NormalizeSelection(selection, backupbrowser.BuildEntries(paths, "", true))
if err != nil {
- return selectionInternal{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)
+ return selectionPlan{}, fmt.Errorf("%w: %w", common.ErrInvalidBackupSelection, err)
}
- return resolved, nil
+ return selectionPlan{
+ entries: entries,
+ archivePaths: kit.Ternary(entry.Format == volume.BackupFormatArchive, paths, nil),
+ }, nil
}
-func archiveMembersForSelectionInternal(paths []string, selected []backuptypes.BackupFileEntry) []string {
+func archiveMembersForSelection(paths []string, selected []backuptypes.BackupFileEntry) []string {
files := make([]string, 0, len(paths))
for _, raw := range paths {
directory := strings.HasSuffix(strings.TrimSpace(raw), "/")
@@ -235,23 +233,18 @@ func archiveMembersForSelectionInternal(paths []string, selected []backuptypes.B
return slices.Compact(files)
}
-func (
- s *Service,
-) restoreSelectionInternal(
+func (s *Service) restoreSelection(
ctx context.Context,
dockerClient *client.Client,
volumeName, backupID string,
entry *volume.Backup,
repository backup.Repository,
snapshotID string,
- selection selectionInternal,
+ selection selectionPlan,
) error {
if entry.Format == volume.BackupFormatArchive {
- if selection.globalRoot {
- return s.restoreArchiveBackupInternal(ctx, dockerClient, volumeName, backupID)
- }
- members := archiveMembersForSelectionInternal(selection.archivePaths, selection.entries)
- return s.restoreArchiveBackupFilesInternal(ctx, dockerClient, volumeName, backupID, members)
+ // nil members restores the whole archive.
+ return s.restoreArchiveBackup(ctx, dockerClient, volumeName, backupID, kit.Ternary(selection.globalRoot, nil, archiveMembersForSelection(selection.archivePaths, selection.entries)))
}
password, err := s.deps.Password(ctx, dockerClient, repository)
if err != nil {
@@ -299,15 +292,15 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID s
return err
}
}
- resolvedSelection, err := s.resolveRestoreSelectionInternal(ctx, dockerClient, entry, repository, snapshotID, selection)
+ resolvedSelection, err := s.resolveRestoreSelection(ctx, dockerClient, entry, repository, snapshotID, selection)
if err != nil {
return err
}
- stopped, err := s.deps.StopContainers(ctx, dockerClient, volumeName, user, true)
+ stopped, err := s.deps.StopContainers(ctx, volumeName, user, true)
containersStopped := len(stopped) > 0
defer func() {
if containersStopped {
- _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ _, restartErr := s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)
err = errors.Join(err, restartErr)
}
}()
@@ -318,7 +311,7 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID s
if err != nil {
return fmt.Errorf("failed to create pre-restore backup: %w", err)
}
- if restoreSelectionErr := s.restoreSelectionInternal(
+ if restoreSelectionErr := s.restoreSelection(
ctx,
dockerClient,
volumeName,
@@ -331,7 +324,7 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID s
return restoreSelectionErr
}
if containersStopped {
- stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ stopped, err = s.deps.StartContainers(context.WithoutCancel(ctx), stopped, user)
containersStopped = len(stopped) > 0
if err != nil {
return err
@@ -353,12 +346,13 @@ func (s *Service) RestoreBackupFiles(ctx context.Context, volumeName, backupID s
return nil
}
-func (s *Service) restoreArchiveBackupInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string) error {
+// restoreArchiveBackup restores a legacy archive into volumeName: all of it when members is nil, otherwise only members.
+func (s *Service) restoreArchiveBackup(ctx context.Context, dockerClient *client.Client, volumeName, backupID string, members []string) error {
filename, err := s.deps.ArchiveFilename(backupID)
if err != nil {
return err
}
- helperImage, err := s.deps.HelperImage(ctx, dockerClient)
+ helperImage, err := s.deps.HelperImage(ctx)
if err != nil {
return err
}
@@ -366,37 +360,52 @@ func (s *Service) restoreArchiveBackupInternal(ctx context.Context, dockerClient
if err != nil {
return err
}
- config := &container.Config{
- Image: helperImage,
- Cmd: []string{
- "sh",
- "-c",
- fmt.Sprintf(
- "set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf "+
- "/backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; "+
- "find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"",
- filename,
- filename,
- ),
- },
- Labels: volumehelper.Labels(),
- }
- hostConfig := volumehelper.HostConfig(helperImage, []string{
- volumeName + ":/volume",
- }, []mount.Mount{backupMount})
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
+ // A full restore runs the replace script as the container command; a partial one execs into a sleeping helper.
+ fullRestoreCmd := []string{
+ "sh",
+ "-c",
+ fmt.Sprintf(
+ "set -e; tmp=$(mktemp -d /volume/.restore_tmp.XXXXXX); tar -tzf /backups/%s >/dev/null; tar -xzf "+
+ "/backups/%s -C \"$tmp\"; find /volume -mindepth 1 -maxdepth 1 -not -path \"$tmp\" -exec rm -rf -- {} +; "+
+ "find \"$tmp\" -mindepth 1 -maxdepth 1 -exec mv -- {} /volume/ \\;; rmdir \"$tmp\"",
+ filename,
+ filename,
+ ),
+ }
+ resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &container.Config{
+ Image: helperImage,
+ Cmd: kit.Ternary(members == nil, fullRestoreCmd, []string{"sleep", "infinity"}),
+ NetworkDisabled: true,
+ Labels: volumehelper.Labels(),
+ },
+ HostConfig: volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume"}, []mount.Mount{backupMount}),
+ })
})
if err != nil {
return fmt.Errorf("failed to create restore container: %w", err)
}
defer func() {
- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ })
}()
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
+ if _, containerStartErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
return fmt.Errorf("failed to start restore container: %w", containerStartErr)
}
+ if members != nil {
+ stderr, restoreErr := s.RestoreArchiveMembers(ctx, resp.ID, filename, members)
+ if restoreErr != nil {
+ return restoreErr
+ }
+ if strings.TrimSpace(stderr) != "" {
+ slog.DebugContext(ctx, "volume service: restore files stderr", "backupId", backupID, "stderr", strings.TrimSpace(stderr))
+ }
+ return nil
+ }
waitResult := dockerClient.ContainerWait(ctx, resp.ID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
var waitBody container.WaitResponse
select {
@@ -416,7 +425,7 @@ func (s *Service) restoreArchiveBackupInternal(ctx context.Context, dockerClient
// container's /volume; the container must mount backup storage at /backups.
func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error) {
args := make([]string, 0, len(cleanedPaths)+5)
- args = append(args, "sh", "-c", restoreBackupFilesScriptInternal, "sh", path.Join("/backups", filename))
+ args = append(args, "sh", "-c", restoreBackupFilesScript, "sh", path.Join("/backups", filename))
for _, cleaned := range cleanedPaths {
args = append(args, "./"+cleaned)
}
@@ -427,51 +436,6 @@ func (s *Service) RestoreArchiveMembers(ctx context.Context, containerID, filena
return stderr, nil
}
-func (s *Service) restoreArchiveBackupFilesInternal(ctx context.Context, dockerClient *client.Client, volumeName, backupID string, cleanedPaths []string) error {
- filename, err := s.deps.ArchiveFilename(backupID)
- if err != nil {
- return err
- }
- helperImage, err := s.deps.HelperImage(ctx, dockerClient)
- if err != nil {
- return err
- }
- backupMount, err := s.deps.StorageMount(ctx, dockerClient, "/backups", true)
- if err != nil {
- return err
- }
- config := &container.Config{
- Image: helperImage,
- Cmd: []string{"sleep", "infinity"},
- NetworkDisabled: true,
- Labels: volumehelper.Labels(),
- }
- hostConfig := volumehelper.HostConfig(helperImage, []string{
- volumeName + ":/volume",
- }, []mount.Mount{backupMount})
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
- })
- if err != nil {
- return fmt.Errorf("failed to create restore container: %w", err)
- }
- defer func() {
- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
- }()
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
- return fmt.Errorf("failed to start restore container: %w", containerStartErr)
- }
- stderr, err := s.RestoreArchiveMembers(ctx, resp.ID, filename, cleanedPaths)
- if err != nil {
- return fmt.Errorf("failed to restore files: %w", err)
- }
- if strings.TrimSpace(stderr) != "" {
- slog.DebugContext(ctx, "volume service: restore files stderr", "backupId", backupID, "stderr", strings.TrimSpace(stderr))
- }
- return nil
-}
-
func (s *Service) UploadAndRestore(ctx context.Context, volumeName string, archive io.ReadSeeker, filename string, user usertypes.Actor) error {
slog.DebugContext(ctx, "volume service: upload and restore", "volume", volumeName, "filename", filename, "user", user.ID)
diff --git a/backend/internal/volume/children/backup/children/restore/service_test.go b/backend/internal/volume/children/backup/children/restore/service_test.go
index 8a6c70938..936e85029 100644
--- a/backend/internal/volume/children/backup/children/restore/service_test.go
+++ b/backend/internal/volume/children/backup/children/restore/service_test.go
@@ -8,13 +8,13 @@ import (
"github.com/stretchr/testify/require"
)
-func TestArchiveMembersForSelectionExpandsFoldersInternal(t *testing.T) {
+func TestArchiveMembersForSelectionExpandsFolders(t *testing.T) {
paths := []string{"./folder/", "./folder/a.txt", "./folder/nested/b.txt", "./other.txt"}
selected := []backup.BackupFileEntry{{Path: "folder", Name: "folder", IsDirectory: true}}
require.Equal(
t,
[]string{"folder/", "folder/a.txt", "folder/nested/b.txt"},
- archiveMembersForSelectionInternal(paths, selected),
+ archiveMembersForSelection(paths, selected),
)
}
@@ -36,10 +36,10 @@ func TestRestoreBackupFilesScriptUsesSupportedTooling(t *testing.T) {
"tar -r",
"$((",
} {
- require.NotContains(t, restoreBackupFilesScriptInternal, unsupported)
+ require.NotContains(t, restoreBackupFilesScript, unsupported)
}
if shellPath, err := exec.LookPath("sh"); err == nil {
- output, syntaxErr := exec.Command(shellPath, "-n", "-c", restoreBackupFilesScriptInternal).CombinedOutput()
+ output, syntaxErr := exec.Command(shellPath, "-n", "-c", restoreBackupFilesScript).CombinedOutput()
require.NoErrorf(t, syntaxErr, "restore script has invalid syntax: %s", output)
}
}
diff --git a/backend/internal/volume/children/backup/helpers.go b/backend/internal/volume/children/backup/helpers.go
index 8dd6fc422..4f1ee4a90 100644
--- a/backend/internal/volume/children/backup/helpers.go
+++ b/backend/internal/volume/children/backup/helpers.go
@@ -56,9 +56,8 @@ func resolveBackupStorageMountFromMounts(ctx context.Context, mounts []container
return mirroredMount
}
-// localRusticRepositoryInitialized reports whether the local volume repository
-// has a Rustic config file. Only a /backups mount in the Arcane container can be
-// checked without starting Rustic, so any other setup counts as initialized.
+// localRusticRepositoryInitialized reports whether the local repository has a Rustic config.
+// Only a /backups mount in the Arcane container can be checked without Rustic; other setups count as initialized.
func (s *Service) localRusticRepositoryInitialized(ctx context.Context, dockerClient *client.Client) bool {
inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
if err != nil || docker.MountForDestination(inspect.Mounts, "/backups", "/backups") == nil {
@@ -78,12 +77,8 @@ func backupMountWarningFromArcaneMounts(mounts []container.MountPoint) string {
}
func (s *Service) backupMountWarning(ctx context.Context) string {
- dockerClient, err := s.deps.Docker.GetClient(ctx)
- if err != nil {
- return ""
- }
// Cannot determine Arcane mount status (e.g. running outside Docker); suppress warning.
- inspect, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
+ inspect, err := s.deps.Docker.Call(ctx, libarcane.InspectCurrentArcaneContainer)
if err != nil {
return ""
}
@@ -102,23 +97,25 @@ func (s *Service) StorageMount(ctx context.Context, dockerClient *client.Client,
}
}
slog.DebugContext(ctx, "volume service: ensure backup volume", "backupVolume", s.deps.BackupVolumeName)
- ensureClient, err := s.deps.Docker.GetClient(ctx)
- if err != nil {
- return mount.Mount{}, err
- }
- if _, inspectErr := ensureClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{}); inspectErr != nil {
- if _, createErr := ensureClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName}); createErr != nil {
+ if _, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, s.deps.BackupVolumeName, client.VolumeInspectOptions{})
+ }); inspectErr != nil {
+ if _, createErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {
+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: s.deps.BackupVolumeName})
+ }); createErr != nil {
return mount.Mount{}, fmt.Errorf("failed to create backup volume: %w", createErr)
}
}
return mount.Mount{Type: mount.TypeVolume, Source: s.deps.BackupVolumeName, Target: target, ReadOnly: readOnly}, nil
}
-func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClient *client.Client, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error) {
+func (s *Service) stopRunningContainersForBackup(ctx context.Context, volumeName string, user usertypes.Actor, refuseArcaneWriters bool) ([]container.Summary, error) {
if s.deps.StopContainer == nil || s.deps.StartContainer == nil {
return nil, errors.New("container service is unavailable")
}
- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ containers, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list running containers before volume backup: %w", err)
}
@@ -151,7 +148,7 @@ func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClie
continue
}
if stopContainerErr := s.deps.StopContainer(ctx, candidate.ID, user); stopContainerErr != nil {
- stillStopped, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ stillStopped, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), stopped, user)
return stillStopped, errors.Join(fmt.Errorf("failed to stop container %s before volume backup: %w", candidate.ID, stopContainerErr), restartErr)
}
stopped = append(stopped, candidate)
@@ -160,14 +157,16 @@ func (s *Service) stopRunningContainersForBackup(ctx context.Context, dockerClie
}
//nolint:gocognit // recovery retries must reconcile IDs, names, and Compose identities in one bounded loop
-func (s *Service) startContainersAfterBackup(ctx context.Context, dockerClient *client.Client, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {
+func (s *Service) startContainersAfterBackup(ctx context.Context, stoppedContainers []container.Summary, user usertypes.Actor) ([]container.Summary, error) {
recoveryCtx, cancel := context.WithTimeout(ctx, volumeBackupContainerRecoveryTimeout)
defer cancel()
remaining := append([]container.Summary(nil), stoppedContainers...)
lastErrors := make(map[string]error, len(stoppedContainers))
for len(remaining) > 0 {
- currentContainers, listErr := dockerClient.ContainerList(recoveryCtx, client.ContainerListOptions{All: true})
+ currentContainers, listErr := s.deps.Docker.Call(recoveryCtx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if listErr == nil {
currentByID := make(map[string]container.Summary, len(currentContainers.Items))
currentByName := make(map[string]container.Summary, len(currentContainers.Items))
@@ -355,24 +354,32 @@ func (s *Service) logBackupEvent(ctx context.Context, eventType event.EventType,
}
}
-func (s *Service) createBackupTempContainerWithMount(ctx context.Context, dockerClient *client.Client, backupMount mount.Mount) (string, func(), error) {
- helperImage, err := s.deps.HelperImage(ctx, dockerClient)
+func (s *Service) createBackupTempContainerWithMount(ctx context.Context, backupMount mount.Mount) (string, func(), error) {
+ helperImage, err := s.deps.HelperImage(ctx)
if err != nil {
return "", nil, err
}
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()},
- HostConfig: volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount}),
+ resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()},
+ HostConfig: volumehelper.HostConfig(helperImage, nil, []mount.Mount{backupMount}),
+ })
})
if err != nil {
return "", nil, fmt.Errorf("failed to create backup temp container: %w", err)
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ if _, containerStartErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
+ _, _ = s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ })
return "", nil, fmt.Errorf("failed to start backup temp container: %w", containerStartErr)
}
return resp.ID, func() {
- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ })
}, nil
}
@@ -388,5 +395,5 @@ func (s *Service) createBackupTempContainer(ctx context.Context, dockerClient *c
if err != nil {
return "", nil, err
}
- return s.createBackupTempContainerWithMount(ctx, dockerClient, backupMount)
+ return s.createBackupTempContainerWithMount(ctx, backupMount)
}
diff --git a/backend/internal/volume/children/backup/helpers_test.go b/backend/internal/volume/children/backup/helpers_test.go
index 200f8e645..10777c43e 100644
--- a/backend/internal/volume/children/backup/helpers_test.go
+++ b/backend/internal/volume/children/backup/helpers_test.go
@@ -165,7 +165,7 @@ func TestBackupMountWarningFromArcaneMounts(t *testing.T) {
}
}
-func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) (*Service, *client.Client) {
+func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) *Service {
t.Helper()
server := httptest.NewServer(handler)
@@ -180,8 +180,12 @@ func setupVolumeBackupLifecycleTest(t *testing.T, handler http.Handler) (*Servic
db := &database.DB{DB: gormDB}
dockerService := docker.NewDockerClientService(t.Context(), db, &config.Config{}, nil).WithClient(dockerClient)
eventService := event.NewEventService(db, &config.Config{}, nil)
- containerService := containerdomain.NewContainerService(eventService, dockerService, nil, nil, nil)
- return NewService(Dependencies{StopContainer: containerService.StopContainer, StartContainer: containerService.StartContainer}), dockerClient
+ containerService := containerdomain.NewContainerService(eventService, dockerService, nil, nil, nil, nil)
+ return NewService(Dependencies{
+ Docker: dockerService,
+ StopContainer: containerService.StopContainer,
+ StartContainer: containerService.StartContainer,
+ })
}
func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsingVolume(t *testing.T) {
@@ -211,14 +215,14 @@ func TestVolumeBackupContainerLifecycleStopsAndRestartsOnlyRunningContainersUsin
}
})
- service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)
+ service := setupVolumeBackupLifecycleTest(t, serverHandler)
actor := user.Actor{ID: "user-1", Username: "tester"}
- stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)
+ stopped, err := service.stopRunningContainersForBackup(t.Context(), "app-data", actor, false)
require.NoError(t, err)
require.Len(t, stopped, 1)
require.Equal(t, "uses-volume", stopped[0].ID)
- remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)
+ remaining, err := service.startContainersAfterBackup(t.Context(), stopped, actor)
require.NoError(t, err)
require.Empty(t, remaining)
require.Equal(t, []string{"stop:uses-volume", "start:uses-volume"}, operations)
@@ -255,9 +259,9 @@ func TestVolumeBackupContainerLifecycleRollsBackStoppedContainersOnStopFailure(t
}
})
- service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)
+ service := setupVolumeBackupLifecycleTest(t, serverHandler)
actor := user.Actor{ID: "user-1", Username: "tester"}
- stillStopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)
+ stillStopped, err := service.stopRunningContainersForBackup(t.Context(), "app-data", actor, false)
require.ErrorContains(t, err, "failed to stop container second")
require.Empty(t, stillStopped)
require.Equal(t, []string{"stop:first", "stop:second", "start:first"}, operations)
@@ -315,14 +319,14 @@ func TestVolumeBackupContainerLifecycleWaitsForRunningComposeReplacement(t *test
}
})
- service, dockerClient := setupVolumeBackupLifecycleTest(t, serverHandler)
+ service := setupVolumeBackupLifecycleTest(t, serverHandler)
actor := user.Actor{ID: "user-1", Username: "tester"}
- stopped, err := service.stopRunningContainersForBackup(t.Context(), dockerClient, "app-data", actor, false)
+ stopped, err := service.stopRunningContainersForBackup(t.Context(), "app-data", actor, false)
require.NoError(t, err)
require.Len(t, stopped, 1)
require.Equal(t, "old-id", stopped[0].ID)
- remaining, err := service.startContainersAfterBackup(t.Context(), dockerClient, stopped, actor)
+ remaining, err := service.startContainersAfterBackup(t.Context(), stopped, actor)
require.NoError(t, err)
require.Empty(t, remaining)
require.Equal(t, []string{"stop:old-id"}, operations)
diff --git a/backend/internal/volume/children/backup/service.go b/backend/internal/volume/children/backup/service.go
index b26856b2f..37b14cfec 100644
--- a/backend/internal/volume/children/backup/service.go
+++ b/backend/internal/volume/children/backup/service.go
@@ -77,7 +77,7 @@ type Dependencies struct {
AlreadyRunning error
StopContainer func(ctx context.Context, containerID string, user usertypes.Actor) error
StartContainer func(ctx context.Context, containerID string, user usertypes.Actor) error
- HelperImage func(ctx context.Context, dockerClient *client.Client) (string, error)
+ HelperImage func(ctx context.Context) (string, error)
AcquireHelper func(ctx context.Context, volumeName string) (string, func(), error)
Exec func(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error)
BackupVolumeName string
@@ -335,11 +335,16 @@ func (s *Service) CreateSystemManagedBackup(ctx context.Context, volumeName stri
}
trigger = cmp.Or(trigger, backuptypes.VolumeBackupTriggerManual)
transient := &policies.VolumeBackupPolicy{
- VolumeName: volumeName, Enabled: true, Schedule: policy.Schedule, RetentionCount: policy.RetentionCount,
- StopContainers: policy.StopContainers, LocalEnabled: policy.LocalEnabled, S3Enabled: policy.S3Enabled,
+ ID: policyID,
+ VolumeName: volumeName,
+ Enabled: true,
+ Schedule: policy.Schedule,
+ RetentionCount: policy.RetentionCount,
+ StopContainers: policy.StopContainers,
+ LocalEnabled: policy.LocalEnabled,
+ S3Enabled: policy.S3Enabled,
S3DestinationID: policy.S3DestinationID,
}
- transient.ID = policyID
plan, err := s.resolveBackupPlan(ctx, volumeName, trigger, volume.CreateBackupRequest{}, transient)
if err != nil {
return nil, err
@@ -359,12 +364,17 @@ func (s *Service) prepareBackup(ctx context.Context, volumeName, trigger, policy
return nil, nil, err
}
entry := &volume.Backup{
- VolumeName: volumeName, CreatedAt: time.Now(), Status: backuptypes.VolumeBackupStatusRunning,
- Trigger: trigger, Destination: backupDestination(plan.LocalEnabled, plan.S3Enabled), Format: volume.BackupFormatRustic,
- S3DestinationID: plan.S3DestinationID, PolicyID: policyID,
- Type: backuptypes.ManagementTypeForPolicy(policyID),
+ ID: fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8]),
+ VolumeName: volumeName,
+ CreatedAt: time.Now(),
+ Status: backuptypes.VolumeBackupStatusRunning,
+ Trigger: trigger,
+ Destination: backupDestination(plan.LocalEnabled, plan.S3Enabled),
+ Format: volume.BackupFormatRustic,
+ S3DestinationID: plan.S3DestinationID,
+ PolicyID: policyID,
+ Type: backuptypes.ManagementTypeForPolicy(policyID),
}
- entry.ID = fmt.Sprintf("%s-%d-%s", volumeName, time.Now().UnixNano(), uuid.New().String()[:8])
if createBackupErr := s.deps.Store.Create(ctx, entry); createBackupErr != nil {
lease.Release(ctx)
return nil, nil, createBackupErr
@@ -420,10 +430,10 @@ func (s *Service) executeBackup(ctx context.Context, entry *volume.Backup, user
// stopped holds the containers still awaiting a restart.
var stopped []container.Summary
if plan.Policy != nil && plan.Policy.StopContainers {
- stopped, err = s.stopRunningContainersForBackup(ctx, dockerClient, volumeName, user, false)
+ stopped, err = s.stopRunningContainersForBackup(ctx, volumeName, user, false)
defer func() {
if len(stopped) > 0 {
- _, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user)
+ _, restartErr := s.startContainersAfterBackup(context.WithoutCancel(ctx), stopped, user)
err = errors.Join(err, restartErr)
}
}()
@@ -435,7 +445,7 @@ func (s *Service) executeBackup(ctx context.Context, entry *volume.Backup, user
return snapshotErr
}
if len(stopped) > 0 {
- if stopped, err = s.startContainersAfterBackup(context.WithoutCancel(ctx), dockerClient, stopped, user); err != nil {
+ if stopped, err = s.startContainersAfterBackup(context.WithoutCancel(ctx), stopped, user); err != nil {
return err
}
}
@@ -741,11 +751,15 @@ func (s *Service) DownloadBackup(ctx context.Context, backupID string, user *use
return nil, 0, passwordErr
}
scratchVolume := "arcane-rustic-download-" + uuid.New().String()
- if _, volumeCreateErr := dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()}); volumeCreateErr != nil {
+ if _, volumeCreateErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {
+ return dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{Name: scratchVolume, Labels: volumehelper.Labels()})
+ }); volumeCreateErr != nil {
return nil, 0, fmt.Errorf("failed to create download scratch volume: %w", volumeCreateErr)
}
removeScratch := func() {
- _, _ = dockerClient.VolumeRemove(context.WithoutCancel(ctx), scratchVolume, client.VolumeRemoveOptions{Force: true})
+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.VolumeRemoveResult, error) {
+ return dockerClient.VolumeRemove(ctx, scratchVolume, client.VolumeRemoveOptions{Force: true})
+ })
}
scratchMount := mount.Mount{Type: mount.TypeVolume, Source: scratchVolume, Target: "/volume"}
restoreOptions := backup.RestoreOptions{DeleteExtra: true}
@@ -754,7 +768,7 @@ func (s *Service) DownloadBackup(ctx context.Context, backupID string, user *use
return nil, 0, fmt.Errorf("failed to restore Rustic snapshot for download: %w", restoreErr)
}
var removeContainer func()
- if containerID, removeContainer, err = s.createBackupTempContainerWithMount(ctx, dockerClient, scratchMount); err != nil {
+ if containerID, removeContainer, err = s.createBackupTempContainerWithMount(ctx, scratchMount); err != nil {
removeScratch()
return nil, 0, err
}
diff --git a/backend/internal/volume/children/workspace/service.go b/backend/internal/volume/children/workspace/service.go
index 78d89ac89..2f8c33dbb 100644
--- a/backend/internal/volume/children/workspace/service.go
+++ b/backend/internal/volume/children/workspace/service.go
@@ -37,7 +37,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
dockerInternal "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
acfsutils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/acfs"
@@ -52,7 +51,7 @@ type Dependencies struct {
AcquireHelper func(ctx context.Context, volumeName string) (string, func(), error)
RequireACFS func(ctx context.Context, volumeName, containerID string) error
Exec func(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error)
- HelperImage func(ctx context.Context, dockerClient *client.Client) (string, error)
+ HelperImage func(ctx context.Context) (string, error)
BackupStorageMount func(ctx context.Context, dockerClient *client.Client, target string, readOnly bool) (mount.Mount, error)
ArchiveFileTarget func(ctx context.Context, volumeName, backupID, rel string) (string, string, error)
RestoreArchiveMembers func(ctx context.Context, containerID, filename string, cleanedPaths []string) (string, error)
@@ -72,7 +71,7 @@ func NewService(deps Dependencies) *Service {
// MaxFileSizeBytes is the largest file the workspace reads or accepts.
func (s *Service) MaxFileSizeBytes() int64 {
- return s.volumeWorkspaceMaxFileSizeBytesInternal()
+ return s.volumeWorkspaceMaxFileSizeBytes()
}
func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*workspace.Workspace, error) {
@@ -84,8 +83,8 @@ func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*w
}()
inspectionStartedAt := time.Now()
- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {
- return nil, classifyVolumeWorkspaceHelperSupportErrorInternal(err)
+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {
+ return nil, classifyVolumeWorkspaceHelperSupportError(err)
}
slog.DebugContext(ctx, "volume workspace inspection completed", "volume", volumeName, "duration", time.Since(inspectionStartedAt))
@@ -101,7 +100,7 @@ func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*w
slog.DebugContext(ctx, "volume workspace helper acquired", "volume", volumeName, "containerId", containerID, "duration", time.Since(helperStartedAt))
scanStartedAt := time.Now()
- tree, err := s.readVolumeWorkspaceFromContainerInternal(ctx, containerID)
+ tree, err := s.readVolumeWorkspaceFromContainer(ctx, containerID)
if err != nil {
return nil, err
}
@@ -124,7 +123,7 @@ func (s *Service) GetVolumeWorkspace(ctx context.Context, volumeName string) (*w
return tree, nil
}
-func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context, containerID string) (*workspace.Workspace, error) {
+func (s *Service) readVolumeWorkspaceFromContainer(ctx context.Context, containerID string) (*workspace.Workspace, error) {
maxDepth := s.deps.MaxDepth
if maxDepth <= 0 {
maxDepth = 50
@@ -163,7 +162,7 @@ func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context,
done <- execErr
}()
- tree, parseErr := decodeVolumeWorkspaceWalkInternal(pipeReader, maxEntries, s.volumeWorkspaceMaxFileSizeBytesInternal())
+ tree, parseErr := decodeVolumeWorkspaceWalk(pipeReader, maxEntries, s.volumeWorkspaceMaxFileSizeBytes())
if parseErr != nil {
cancel()
_ = pipeReader.CloseWithError(parseErr)
@@ -171,7 +170,7 @@ func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context,
execErr := <-done
_ = pipeReader.Close()
if execErr != nil && (parseErr == nil || ctx.Err() != nil || !errors.Is(execErr, context.Canceled)) {
- return nil, classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace")
+ return nil, classifyVolumeWorkspaceExecError(execErr, stderr.String(), "read volume workspace")
}
if parseErr != nil {
return nil, fmt.Errorf("parse volume workspace: %w", parseErr)
@@ -180,7 +179,7 @@ func (s *Service) readVolumeWorkspaceFromContainerInternal(ctx context.Context,
return tree, nil
}
-func (s *Service) volumeWorkspaceMaxFileSizeBytesInternal() int64 {
+func (s *Service) volumeWorkspaceMaxFileSizeBytes() int64 {
if s.deps.MaxFileSizeBytes <= 0 {
return workspacepkg.MaxFileSizeBytes(workspacepkg.DefaultMaxFileSizeMB)
}
@@ -190,8 +189,8 @@ func (s *Service) volumeWorkspaceMaxFileSizeBytesInternal() int64 {
func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relativePath string) (*workspace.FileContent, error) {
defer s.deps.Locks.RLock(volumeName)()
- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {
- return nil, classifyVolumeWorkspaceHelperSupportErrorInternal(err)
+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {
+ return nil, classifyVolumeWorkspaceHelperSupportError(err)
}
rel, err := kit.NormalizeRelativePath(relativePath)
if err != nil {
@@ -209,7 +208,7 @@ func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relati
"acfs", "stat", "--root", "/volume", "--path", "/" + rel,
})
if err != nil {
- return nil, classifyVolumeWorkspaceExecErrorInternal(err, stderr, "inspect volume workspace file")
+ return nil, classifyVolumeWorkspaceExecError(err, stderr, "inspect volume workspace file")
}
var statResponse types.StatResponse
if unmarshalErr := json.Unmarshal([]byte(stdout), &statResponse); unmarshalErr != nil {
@@ -226,18 +225,18 @@ func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relati
return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, errors.New("path is a directory"))
}
if !acfsutils.IsRegular(entry) {
- return volumeWorkspaceFileContentResponseInternal(rel, "special", entry.Size, nil, s.volumeWorkspaceMaxFileSizeBytesInternal())
+ return volumeWorkspaceFileContentResponse(rel, "special", entry.Size, nil, s.volumeWorkspaceMaxFileSizeBytes())
}
- maxFileSizeBytes := s.volumeWorkspaceMaxFileSizeBytesInternal()
+ maxFileSizeBytes := s.volumeWorkspaceMaxFileSizeBytes()
if entry.Size > maxFileSizeBytes {
- return volumeWorkspaceFileContentResponseInternal(rel, "regular", entry.Size, nil, maxFileSizeBytes)
+ return volumeWorkspaceFileContentResponse(rel, "regular", entry.Size, nil, maxFileSizeBytes)
}
previewLimit := maxFileSizeBytes
if previewLimit < int64(^uint64(0)>>1) {
previewLimit++
}
- reader, size, err := s.startVolumeWorkspaceReadInternal(ctx, containerID, rel, previewLimit, func() {})
+ reader, size, err := s.startVolumeWorkspaceRead(ctx, containerID, rel, previewLimit, func() {})
if err != nil {
return nil, fmt.Errorf("read volume workspace file: %w", err)
}
@@ -250,16 +249,16 @@ func (s *Service) GetVolumeWorkspaceFile(ctx context.Context, volumeName, relati
return nil, fmt.Errorf("close volume workspace file content: %w", closeErr)
}
if size > maxFileSizeBytes {
- return volumeWorkspaceFileContentResponseInternal(rel, "regular", size, nil, maxFileSizeBytes)
+ return volumeWorkspaceFileContentResponse(rel, "regular", size, nil, maxFileSizeBytes)
}
- return volumeWorkspaceFileContentResponseInternal(rel, "regular", size, content, maxFileSizeBytes)
+ return volumeWorkspaceFileContentResponse(rel, "regular", size, content, maxFileSizeBytes)
}
func (s *Service) DownloadVolumeWorkspaceFile(ctx context.Context, volumeName, relativePath string) (io.ReadCloser, int64, error) {
unlock := s.deps.Locks.RLock(volumeName)
- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {
+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {
unlock()
- return nil, 0, classifyVolumeWorkspaceHelperSupportErrorInternal(err)
+ return nil, 0, classifyVolumeWorkspaceHelperSupportError(err)
}
rel, err := kit.NormalizeRelativePath(relativePath)
if err != nil {
@@ -279,10 +278,10 @@ func (s *Service) DownloadVolumeWorkspaceFile(ctx context.Context, volumeName, r
cleanupAndUnlock()
return nil, 0, requireVolumeHelperACFSErr
}
- return s.startVolumeWorkspaceReadInternal(ctx, containerID, rel, 0, cleanupAndUnlock)
+ return s.startVolumeWorkspaceRead(ctx, containerID, rel, 0, cleanupAndUnlock)
}
-type volumeWorkspaceReadStreamInternal struct {
+type volumeWorkspaceReadStream struct {
ctx context.Context
cancel context.CancelFunc
stopClose func() bool
@@ -294,7 +293,7 @@ type volumeWorkspaceReadStreamInternal struct {
err error
}
-func (r *volumeWorkspaceReadStreamInternal) Read(buffer []byte) (int, error) {
+func (r *volumeWorkspaceReadStream) Read(buffer []byte) (int, error) {
if r.remaining == 0 {
var extra [1]byte
read, readErr := r.pipe.Read(extra[:])
@@ -321,7 +320,7 @@ func (r *volumeWorkspaceReadStreamInternal) Read(buffer []byte) (int, error) {
return read, err
}
-func (r *volumeWorkspaceReadStreamInternal) Close() error {
+func (r *volumeWorkspaceReadStream) Close() error {
r.once.Do(func() {
r.stopClose()
r.cancel()
@@ -335,7 +334,7 @@ func (r *volumeWorkspaceReadStreamInternal) Close() error {
return r.err
}
-func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containerID, relativePath string, limit int64, cleanup func()) (io.ReadCloser, int64, error) {
+func (s *Service) startVolumeWorkspaceRead(ctx context.Context, containerID, relativePath string, limit int64, cleanup func()) (io.ReadCloser, int64, error) {
dockerClient, err := s.deps.Docker.GetClient(ctx)
if err != nil {
cleanup()
@@ -361,7 +360,7 @@ func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containe
execErr = fmt.Errorf("acfs read exited with code %d", exitCode)
}
if execErr != nil {
- execErr = classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace file")
+ execErr = classifyVolumeWorkspaceExecError(execErr, stderr.String(), "read volume workspace file")
}
stopClose()
_ = pipeWriter.CloseWithError(execErr)
@@ -389,7 +388,7 @@ func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containe
return nil, 0, errors.New("volume workspace file is too large")
}
size := int64(payloadSize)
- return &volumeWorkspaceReadStreamInternal{
+ return &volumeWorkspaceReadStream{
ctx: ctx,
cancel: cancel,
stopClose: stopClose,
@@ -400,12 +399,10 @@ func (s *Service) startVolumeWorkspaceReadInternal(ctx context.Context, containe
}, size, nil
}
-func (s *Service) validateVolumeHelperSupportInternal(ctx context.Context, volumeName string) error {
- dockerClient, err := s.deps.Docker.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to connect to Docker: %w", err)
- }
- result, err := dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{})
+func (s *Service) validateVolumeHelperSupport(ctx context.Context, volumeName string) error {
+ result, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, volumeName, client.VolumeInspectOptions{})
+ })
if err != nil {
return fmt.Errorf("failed to inspect volume: %w", err)
}
@@ -415,17 +412,8 @@ func (s *Service) validateVolumeHelperSupportInternal(ctx context.Context, volum
return nil
}
-func (s *Service) validateVolumeWorkspacePathInternal(ctx context.Context, containerID, relativePath string, allowMissing bool) error {
- allowMissingValue := kit.Ternary(allowMissing, "1", "0")
- _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceValidatePathScriptInternal, "sh", relativePath, allowMissingValue})
- if err != nil {
- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "validate volume workspace path")
- }
- return nil
-}
-
const (
- volumeWorkspaceValidatePathScriptInternal = `set -e
+ volumeWorkspaceValidatePathScript = `set -e
rel="$1"
allow_missing="$2"
cur=/volume
@@ -450,7 +438,7 @@ while :; do
esac
done`
- volumeWorkspaceBackupCreateScriptInternal = `set -e
+ volumeWorkspaceBackupCreateScript = `set -e
rel="$1"
archive="$2"
cur=/volume
@@ -484,18 +472,13 @@ tar -cf "$archive" "./$entry"
printf 'present\0'`
)
-func (
- s *Service,
-) UpdateVolumeWorkspace(
+func (s *Service) UpdateVolumeWorkspace(
ctx context.Context,
volumeName string,
manifest volume.WorkspaceUpdateManifest,
uploads map[int][]byte,
user usertypes.Actor,
-) (
- *workspace.Workspace,
- error,
-) {
+) (*workspace.Workspace, error) {
totalStartedAt := time.Now()
defer func() {
slog.DebugContext(ctx, "volume workspace update completed", "volume", volumeName, "fileChangeCount", len(manifest.FileChanges), "totalDuration", time.Since(totalStartedAt))
@@ -507,7 +490,7 @@ func (
uploadReferences := make([]workspacepkg.UploadReference, 0, len(manifest.FileChanges))
for _, change := range manifest.FileChanges {
uploadReferences = append(uploadReferences, workspacepkg.UploadReference{Operation: change.Operation, UploadIndex: change.UploadIndex, BaselineIndex: change.BaselineIndex})
- if err := validateVolumeWorkspaceFileChangeInternal(change); err != nil {
+ if err := validateVolumeWorkspaceFileChange(change); err != nil {
return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, err)
}
}
@@ -515,8 +498,8 @@ func (
return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, err)
}
inspectionStartedAt := time.Now()
- if err := s.validateVolumeHelperSupportInternal(ctx, volumeName); err != nil {
- return nil, classifyVolumeWorkspaceHelperSupportErrorInternal(err)
+ if err := s.validateVolumeHelperSupport(ctx, volumeName); err != nil {
+ return nil, classifyVolumeWorkspaceHelperSupportError(err)
}
slog.DebugContext(ctx, "volume workspace inspection completed", "volume", volumeName, "duration", time.Since(inspectionStartedAt))
defer s.deps.Locks.Lock(volumeName)()
@@ -525,7 +508,7 @@ func (
return change.Operation == volume.FileOpRestoreFile
})
helperStartedAt := time.Now()
- containerID, cleanup, err := s.createVolumeWorkspaceMutationContainerInternal(ctx, volumeName, needsBackups)
+ containerID, cleanup, err := s.createVolumeWorkspaceMutationContainer(ctx, volumeName, needsBackups)
if err != nil {
return nil, err
}
@@ -536,7 +519,7 @@ func (
slog.DebugContext(ctx, "volume workspace mutation helper acquired", "volume", volumeName, "containerId", containerID, "dedicated", needsBackups, "duration", time.Since(helperStartedAt))
revisionScanStartedAt := time.Now()
- current, err := s.readVolumeWorkspaceFromContainerInternal(ctx, containerID)
+ current, err := s.readVolumeWorkspaceFromContainer(ctx, containerID)
if err != nil {
return nil, err
}
@@ -556,8 +539,8 @@ func (
revisionScanStartedAt,
),
)
- if validateVolumeWorkspaceRevisionErr := validateVolumeWorkspaceRevisionInternal(manifest.FileTreeRevision, current.FileTreeRevision); validateVolumeWorkspaceRevisionErr != nil {
- return nil, validateVolumeWorkspaceRevisionErr
+ if strings.TrimSpace(manifest.FileTreeRevision) != current.FileTreeRevision {
+ return nil, common.Classify(common.ErrVolumeWorkspaceConflict, errors.New("volume workspace changed; refresh it and try again"))
}
stagingStartedAt := time.Now()
@@ -565,31 +548,31 @@ func (
if err != nil {
return nil, err
}
- identity := s.resolveVolumeWorkspaceWriteIdentityInternal(ctx, dockerClient, containerID, volumeName)
- stagedFiles, err := s.stageVolumeWorkspaceChangesInternal(ctx, dockerClient, containerID, manifest.FileChanges, uploads, identity)
+ identity := s.resolveVolumeWorkspaceWriteIdentity(ctx, containerID, volumeName)
+ stagedFiles, err := s.stageVolumeWorkspaceChanges(ctx, dockerClient, containerID, manifest.FileChanges, uploads, identity)
if err != nil {
return nil, err
}
slog.DebugContext(ctx, "volume workspace staging completed", "volume", volumeName, "stagedFileCount", len(stagedFiles), "duration", time.Since(stagingStartedAt))
backupStartedAt := time.Now()
- scope, err := volumeWorkspaceBackupScopeInternal(manifest.FileChanges)
+ scope, err := volumeWorkspaceBackupScope(manifest.FileChanges)
if err != nil {
return nil, common.Classify(common.ErrVolumeWorkspaceBadRequest, err)
}
for _, relativePath := range scope {
- if validateVolumeWorkspacePathErr := s.validateVolumeWorkspacePathInternal(ctx, containerID, relativePath, true); validateVolumeWorkspacePathErr != nil {
- return nil, validateVolumeWorkspacePathErr
+ if _, stderr, validateErr := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceValidatePathScript, "sh", relativePath, "1"}); validateErr != nil {
+ return nil, classifyVolumeWorkspaceExecError(validateErr, stderr, "validate volume workspace path")
}
}
- backup, err := s.backupVolumeWorkspaceScopeInternal(ctx, containerID, scope)
+ backup, err := s.backupVolumeWorkspaceScope(ctx, containerID, scope)
if err != nil {
return nil, err
}
slog.DebugContext(ctx, "volume workspace backup completed", "volume", volumeName, "scopeCount", len(scope), "duration", time.Since(backupStartedAt))
applyStartedAt := time.Now()
- if applyVolumeWorkspaceChangesErr := s.applyVolumeWorkspaceChangesInternal(
+ if applyVolumeWorkspaceChangesErr := s.applyVolumeWorkspaceChanges(
ctx,
dockerClient,
containerID,
@@ -597,14 +580,30 @@ func (
manifest.FileChanges,
stagedFiles,
identity,
- func() error { return s.restoreVolumeWorkspaceScopeInternal(ctx, containerID, backup) },
+ func() error {
+ args := append([]string{"sh", "-c", `set -e
+for rel do rm -rf -- "/volume/$rel"; done`, "sh"}, volumeWorkspaceRollbackPaths(backup)...)
+ if _, stderr, execErr := s.deps.Exec(ctx, containerID, "", args); execErr != nil {
+ return classifyVolumeWorkspaceExecError(execErr, stderr, "remove failed volume workspace changes")
+ }
+ for _, archive := range backup.archives {
+ containerParent := path.Join("/volume", path.Dir(archive.relativePath))
+ if _, stderr, execErr := s.deps.Exec(ctx, containerID, "", []string{"mkdir", "-p", containerParent}); execErr != nil {
+ return classifyVolumeWorkspaceExecError(execErr, stderr, "recreate volume workspace backup parent")
+ }
+ if _, stderr, execErr := s.deps.Exec(ctx, containerID, "", []string{"tar", "-xf", archive.archivePath, "-C", containerParent}); execErr != nil {
+ return classifyVolumeWorkspaceExecError(execErr, stderr, "restore volume workspace backup")
+ }
+ }
+ return nil
+ },
); applyVolumeWorkspaceChangesErr != nil {
return nil, applyVolumeWorkspaceChangesErr
}
slog.DebugContext(ctx, "volume workspace changes applied", "volume", volumeName, "fileChangeCount", len(manifest.FileChanges), "duration", time.Since(applyStartedAt))
finalScanStartedAt := time.Now()
- tree, err := s.readVolumeWorkspaceFromContainerInternal(ctx, containerID)
+ tree, err := s.readVolumeWorkspaceFromContainer(ctx, containerID)
if err != nil {
return nil, err
}
@@ -634,52 +633,56 @@ func (
return tree, nil
}
-// volumeWorkspaceWriteIdentityInternal is the identity workspace file mutations run as,
+// volumeWorkspaceWriteIdentity is the identity workspace file mutations run as,
// so files land owned by the user the consuming container runs with. The zero value means root.
-type volumeWorkspaceWriteIdentityInternal struct {
+type volumeWorkspaceWriteIdentity struct {
execUser string
uid int
gid int
}
-func volumeWorkspaceWriteIdentityFromConfigUserInternal(configUser string) volumeWorkspaceWriteIdentityInternal {
+func volumeWorkspaceWriteIdentityFromConfigUser(configUser string) volumeWorkspaceWriteIdentity {
trimmed := strings.TrimSpace(configUser)
if trimmed == "" {
- return volumeWorkspaceWriteIdentityInternal{}
+ return volumeWorkspaceWriteIdentity{}
}
// Named users cannot be resolved against the helper image, and uid 0 is already the default.
uidPart, gidPart, hasGID := strings.Cut(trimmed, ":")
uid, err := strconv.ParseUint(uidPart, 10, 32)
if err != nil || uid == 0 {
- return volumeWorkspaceWriteIdentityInternal{}
+ return volumeWorkspaceWriteIdentity{}
}
var gid uint64
if hasGID {
gid, err = strconv.ParseUint(gidPart, 10, 32)
if err != nil {
- return volumeWorkspaceWriteIdentityInternal{}
+ return volumeWorkspaceWriteIdentity{}
}
}
- return volumeWorkspaceWriteIdentityInternal{execUser: trimmed, uid: int(uid), gid: int(gid)}
+ return volumeWorkspaceWriteIdentity{execUser: trimmed, uid: int(uid), gid: int(gid)}
}
-func (s *Service) resolveVolumeWorkspaceWriteIdentityInternal(ctx context.Context, dockerClient *client.Client, containerID, volumeName string) volumeWorkspaceWriteIdentityInternal {
- consumerIDs, err := docker.GetContainersUsingVolume(ctx, dockerClient, volumeName)
+func (s *Service) resolveVolumeWorkspaceWriteIdentity(ctx context.Context, containerID, volumeName string) volumeWorkspaceWriteIdentity {
+ consumerIDs, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]string, error) {
+ return docker.GetContainersUsingVolume(ctx, dockerClient, volumeName)
+ })
if err != nil {
slog.WarnContext(ctx, "could not list containers for volume workspace write identity", "volume", volumeName, "error", err.Error())
}
- identity := volumeWorkspaceWriteIdentityInternal{}
+ identity := volumeWorkspaceWriteIdentity{}
identitySource := ""
for _, consumerID := range consumerIDs {
- inspect, inspectErr := compat.ContainerInspectWithCompatibility(ctx, dockerClient, consumerID, client.ContainerInspectOptions{})
+ inspect, inspectErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, consumerID, client.ContainerInspectOptions{})
+ })
if inspectErr != nil || inspect.Container.Config == nil {
// An uninspectable consumer could declare a conflicting user, so the identity is
// indeterminate; defer to the volume root's owner.
slog.WarnContext(ctx, "could not inspect volume consumer; using volume root owner for workspace writes", "volume", volumeName, "containerId", consumerID)
- identity = volumeWorkspaceWriteIdentityInternal{}
+ identity = volumeWorkspaceWriteIdentity{}
break
}
- parsed := volumeWorkspaceWriteIdentityFromConfigUserInternal(inspect.Container.Config.User)
+ parsed := volumeWorkspaceWriteIdentityFromConfigUser(inspect.Container.Config.User)
if parsed.execUser == "" {
continue
}
@@ -700,7 +703,7 @@ func (s *Service) resolveVolumeWorkspaceWriteIdentityInternal(ctx context.Contex
"conflictingIdentity",
parsed.execUser,
)
- identity = volumeWorkspaceWriteIdentityInternal{}
+ identity = volumeWorkspaceWriteIdentity{}
break
}
}
@@ -712,45 +715,40 @@ func (s *Service) resolveVolumeWorkspaceWriteIdentityInternal(ctx context.Contex
// volume root is the next best signal for the runtime identity.
stdout, _, err := s.deps.Exec(ctx, containerID, "", []string{"stat", "-c", "%u:%g", "/volume"})
if err != nil {
- return volumeWorkspaceWriteIdentityInternal{}
+ return volumeWorkspaceWriteIdentity{}
}
- identity = volumeWorkspaceWriteIdentityFromConfigUserInternal(stdout)
+ identity = volumeWorkspaceWriteIdentityFromConfigUser(stdout)
if identity.execUser != "" {
slog.InfoContext(ctx, "volume workspace writes run as volume root owner", "volume", volumeName, "identity", identity.execUser)
}
return identity
}
-type volumeWorkspaceStagedFileInternal struct {
+type volumeWorkspaceStagedFile struct {
path string
size int64
}
-type volumeWorkspaceStagedContentInternal struct {
+type volumeWorkspaceStagedContent struct {
name string
content io.ReadCloser
size int64
}
-func (
- s *Service,
-) stageVolumeWorkspaceChangesInternal(
+func (s *Service) stageVolumeWorkspaceChanges(
ctx context.Context,
dockerClient *client.Client,
containerID string,
changes []volume.WorkspaceFileChange,
uploads map[int][]byte,
- identity volumeWorkspaceWriteIdentityInternal,
-) (
- map[int]volumeWorkspaceStagedFileInternal,
- error,
-) {
+ identity volumeWorkspaceWriteIdentity,
+) (map[int]volumeWorkspaceStagedFile, error) {
if _, _, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", "rm -rf -- /tmp/arcane-workspace && mkdir -p -- /tmp/arcane-workspace"}); err != nil {
return nil, fmt.Errorf("prepare volume workspace staging directory: %w", err)
}
- stagedFiles := make(map[int]volumeWorkspaceStagedFileInternal)
- stagedContents := make([]volumeWorkspaceStagedContentInternal, 0, len(changes))
+ stagedFiles := make(map[int]volumeWorkspaceStagedFile)
+ stagedContents := make([]volumeWorkspaceStagedContent, 0, len(changes))
for index, change := range changes {
if change.UploadIndex == nil {
continue
@@ -759,8 +757,8 @@ func (
content := uploads[*change.UploadIndex]
reader := io.NopCloser(bytes.NewReader(content))
size := int64(len(content))
- stagedFiles[index] = volumeWorkspaceStagedFileInternal{path: stagedPath, size: size}
- stagedContents = append(stagedContents, volumeWorkspaceStagedContentInternal{
+ stagedFiles[index] = volumeWorkspaceStagedFile{path: stagedPath, size: size}
+ stagedContents = append(stagedContents, volumeWorkspaceStagedContent{
name: path.Base(stagedPath),
content: reader,
size: size,
@@ -769,23 +767,23 @@ func (
if len(stagedContents) == 0 {
return stagedFiles, nil
}
- if err := s.copyVolumeWorkspaceFilesToContainerInternal(ctx, dockerClient, containerID, stagedContents, identity); err != nil {
+ if err := s.copyVolumeWorkspaceFilesToContainer(ctx, dockerClient, containerID, stagedContents, identity); err != nil {
return nil, err
}
return stagedFiles, nil
}
-func (s *Service) applyVolumeWorkspaceChangesInternal(
+func (s *Service) applyVolumeWorkspaceChanges(
ctx context.Context,
dockerClient *client.Client,
containerID string,
volumeName string,
changes []volume.WorkspaceFileChange,
- stagedFiles map[int]volumeWorkspaceStagedFileInternal,
- identity volumeWorkspaceWriteIdentityInternal,
+ stagedFiles map[int]volumeWorkspaceStagedFile,
+ identity volumeWorkspaceWriteIdentity,
rollback func() error,
) error {
- rollbackFailureInternal := func(applyErr error) error {
+ rollbackFailure := func(applyErr error) error {
if rollbackErr := rollback(); rollbackErr != nil {
return errors.Join(applyErr, fmt.Errorf("failed to roll back volume workspace: %w", rollbackErr))
}
@@ -795,8 +793,19 @@ func (s *Service) applyVolumeWorkspaceChangesInternal(
for index := 0; index < len(changes); {
if changes[index].Operation == volume.FileOpRestoreFile {
rel, _ := kit.NormalizeRelativePath(changes[index].RelativePath)
- if err := s.restoreVolumeWorkspaceFileInternal(ctx, containerID, volumeName, changes[index].BackupID, rel); err != nil {
- return rollbackFailureInternal(err)
+ if _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceValidatePathScript, "sh", rel, "1"}); err != nil {
+ return rollbackFailure(classifyVolumeWorkspaceExecError(err, stderr, "validate volume workspace path"))
+ }
+ filename, cleaned, err := s.deps.ArchiveFileTarget(ctx, volumeName, changes[index].BackupID, rel)
+ if err != nil {
+ return rollbackFailure(err)
+ }
+ stderr, err := s.deps.RestoreArchiveMembers(ctx, containerID, filename, []string{cleaned})
+ if err != nil {
+ return rollbackFailure(classifyVolumeWorkspaceExecError(err, stderr, "restore volume workspace file"))
+ }
+ if strings.TrimSpace(stderr) != "" {
+ slog.DebugContext(ctx, "volume workspace change stderr", "operation", volume.FileOpRestoreFile, "path", rel, "stderr", strings.TrimSpace(stderr))
}
index++
continue
@@ -806,26 +815,26 @@ func (s *Service) applyVolumeWorkspaceChangesInternal(
for end < len(changes) && changes[end].Operation != volume.FileOpRestoreFile {
end++
}
- if err := s.executeVolumeWorkspaceACFSBatchInternal(ctx, dockerClient, containerID, changes[index:end], stagedFiles, index, identity); err != nil {
- return rollbackFailureInternal(err)
+ if err := s.executeVolumeWorkspaceACFSBatch(ctx, dockerClient, containerID, changes[index:end], stagedFiles, index, identity); err != nil {
+ return rollbackFailure(err)
}
index = end
}
return nil
}
-func (s *Service) executeVolumeWorkspaceACFSBatchInternal(
+func (s *Service) executeVolumeWorkspaceACFSBatch(
ctx context.Context,
dockerClient *client.Client,
containerID string,
changes []volume.WorkspaceFileChange,
- stagedFiles map[int]volumeWorkspaceStagedFileInternal,
+ stagedFiles map[int]volumeWorkspaceStagedFile,
startIndex int,
- identity volumeWorkspaceWriteIdentityInternal,
+ identity volumeWorkspaceWriteIdentity,
) error {
applyChanges := make([]types.ApplyChange, 0, len(changes))
for offset, change := range changes {
- applyChange, err := mapVolumeWorkspaceChangeToACFSInternal(change, stagedFiles[startIndex+offset])
+ applyChange, err := mapVolumeWorkspaceChangeToACFS(change, stagedFiles[startIndex+offset])
if err != nil {
return common.Classify(common.ErrVolumeWorkspaceBadRequest, err)
}
@@ -840,7 +849,7 @@ func (s *Service) executeVolumeWorkspaceACFSBatchInternal(
return fmt.Errorf("encode volume workspace apply manifest: %w", err)
}
manifestName := fmt.Sprintf("manifest-%d.json", startIndex)
- if copyVolumeWorkspaceFilesToContainerErr := s.copyVolumeWorkspaceFilesToContainerInternal(ctx, dockerClient, containerID, []volumeWorkspaceStagedContentInternal{{
+ if copyVolumeWorkspaceFilesToContainerErr := s.copyVolumeWorkspaceFilesToContainer(ctx, dockerClient, containerID, []volumeWorkspaceStagedContent{{
name: manifestName,
content: io.NopCloser(bytes.NewReader(manifestContent)),
size: int64(len(manifestContent)),
@@ -852,7 +861,7 @@ func (s *Service) executeVolumeWorkspaceACFSBatchInternal(
"acfs", "apply", "--root", "/volume", "--staging", "/tmp/arcane-workspace", "--manifest", manifestName,
})
if err != nil {
- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "apply volume workspace changes")
+ return classifyVolumeWorkspaceExecError(err, stderr, "apply volume workspace changes")
}
var response types.ApplyResponse
if unmarshalErr := json.Unmarshal([]byte(stdout), &response); unmarshalErr != nil {
@@ -864,59 +873,61 @@ func (s *Service) executeVolumeWorkspaceACFSBatchInternal(
return nil
}
-func mapVolumeWorkspaceChangeToACFSInternal(change volume.WorkspaceFileChange, staged volumeWorkspaceStagedFileInternal) (types.ApplyChange, error) {
+func mapVolumeWorkspaceChangeToACFS(change volume.WorkspaceFileChange, staged volumeWorkspaceStagedFile) (types.ApplyChange, error) {
relativePath, err := kit.NormalizeRelativePath(change.RelativePath)
if err != nil {
return types.ApplyChange{}, err
}
- result := types.ApplyChange{Path: "/" + relativePath, Recursive: change.Recursive}
switch change.Operation {
- case volume.FileOpCreateFile:
- result.Operation = types.ApplyCreateFile
- result.StagedName = path.Base(staged.path)
- result.Size = staged.size
- case volume.FileOpUpdateFile:
- result.Operation = types.ApplyUpdateFile
- result.StagedName = path.Base(staged.path)
- result.Size = staged.size
+ case volume.FileOpCreateFile, volume.FileOpUpdateFile:
+ return types.ApplyChange{
+ Operation: kit.Ternary(change.Operation == volume.FileOpCreateFile, types.ApplyCreateFile, types.ApplyUpdateFile),
+ Path: "/" + relativePath,
+ StagedName: path.Base(staged.path),
+ Size: staged.size,
+ Recursive: change.Recursive,
+ }, nil
case volume.FileOpCreateFolder:
- result.Operation = types.ApplyCreateFolder
- result.Recursive = false
+ return types.ApplyChange{Operation: types.ApplyCreateFolder, Path: "/" + relativePath}, nil
case volume.FileOpRename:
newName, nameErr := kit.ValidateFileName(change.NewName)
if nameErr != nil {
return types.ApplyChange{}, nameErr
}
- result.Operation = types.ApplyRename
- result.TargetPath = "/" + path.Join(path.Dir(relativePath), newName)
- result.Recursive = false
+ return types.ApplyChange{
+ Operation: types.ApplyRename,
+ Path: "/" + relativePath,
+ TargetPath: "/" + path.Join(path.Dir(relativePath), newName),
+ }, nil
case volume.FileOpMove:
parent := ""
if strings.TrimSpace(change.NewParentPath) != "" {
- parent, err = kit.NormalizeRelativePath(change.NewParentPath)
- if err != nil {
+ if parent, err = kit.NormalizeRelativePath(change.NewParentPath); err != nil {
return types.ApplyChange{}, err
}
}
- result.Operation = types.ApplyMove
- result.TargetPath = "/" + path.Join(parent, path.Base(relativePath))
- result.Recursive = false
+ return types.ApplyChange{
+ Operation: types.ApplyMove,
+ Path: "/" + relativePath,
+ TargetPath: "/" + path.Join(parent, path.Base(relativePath)),
+ }, nil
case volume.FileOpDelete:
- result.Operation = types.ApplyDelete
+ return types.ApplyChange{
+ Operation: types.ApplyDelete,
+ Path: "/" + relativePath,
+ Recursive: change.Recursive,
+ }, nil
default:
return types.ApplyChange{}, fmt.Errorf("unsupported volume workspace operation %q", change.Operation)
}
- return result, nil
}
-func (
- s *Service,
-) copyVolumeWorkspaceFilesToContainerInternal(
+func (s *Service) copyVolumeWorkspaceFilesToContainer(
ctx context.Context,
dockerClient *client.Client,
containerID string,
- contents []volumeWorkspaceStagedContentInternal,
- identity volumeWorkspaceWriteIdentityInternal,
+ contents []volumeWorkspaceStagedContent,
+ identity volumeWorkspaceWriteIdentity,
) error {
pipeReader, pipeWriter := io.Pipe()
archiveDone := make(chan error, 1)
@@ -952,7 +963,7 @@ func (
return nil
}
-func (s *Service) createVolumeWorkspaceMutationContainerInternal(ctx context.Context, volumeName string, withBackups bool) (string, func(), error) {
+func (s *Service) createVolumeWorkspaceMutationContainer(ctx context.Context, volumeName string, withBackups bool) (string, func(), error) {
if !withBackups {
return s.deps.AcquireHelper(ctx, volumeName)
}
@@ -960,7 +971,7 @@ func (s *Service) createVolumeWorkspaceMutationContainerInternal(ctx context.Con
if err != nil {
return "", nil, err
}
- helperImage, err := s.deps.HelperImage(ctx, dockerClient)
+ helperImage, err := s.deps.HelperImage(ctx)
if err != nil {
return "", nil, err
}
@@ -970,52 +981,58 @@ func (s *Service) createVolumeWorkspaceMutationContainerInternal(ctx context.Con
}
config := &container.Config{Image: helperImage, Cmd: []string{"sleep", "infinity"}, NetworkDisabled: true, Labels: volumehelper.Labels()}
hostConfig := volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume"}, []mount.Mount{backupMount})
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: config, HostConfig: hostConfig})
+ resp, err := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{Config: config, HostConfig: hostConfig})
+ })
if err != nil {
return "", nil, fmt.Errorf("create volume workspace helper: %w", err)
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
+ if _, containerStartErr := s.deps.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
startErr := fmt.Errorf("start volume workspace helper: %w", containerStartErr)
- cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)
- defer cancel()
- if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {
+ if _, cleanupErr := s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ }); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {
return "", nil, errors.Join(startErr, fmt.Errorf("remove volume workspace helper after start failure: %w", cleanupErr))
}
return "", nil, startErr
}
return resp.ID, func() {
- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ _, _ = s.deps.Docker.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ })
}, nil
}
-type volumeWorkspaceBackupArchiveInternal struct {
+type volumeWorkspaceBackupArchive struct {
relativePath string
archivePath string
}
-type volumeWorkspaceBackupInternal struct {
- archives []volumeWorkspaceBackupArchiveInternal
+type volumeWorkspaceBackup struct {
+ archives []volumeWorkspaceBackupArchive
absentEntries []string
}
-func (s *Service) backupVolumeWorkspaceScopeInternal(ctx context.Context, containerID string, scope []string) (*volumeWorkspaceBackupInternal, error) {
+func (s *Service) backupVolumeWorkspaceScope(ctx context.Context, containerID string, scope []string) (*volumeWorkspaceBackup, error) {
if _, _, err := s.deps.Exec(ctx, containerID, "", []string{"mkdir", "-p", "/tmp/arcane-workspace"}); err != nil {
return nil, fmt.Errorf("prepare volume workspace backup directory: %w", err)
}
- backup := &volumeWorkspaceBackupInternal{
- archives: make([]volumeWorkspaceBackupArchiveInternal, 0, len(scope)),
+ backup := &volumeWorkspaceBackup{
+ archives: make([]volumeWorkspaceBackupArchive, 0, len(scope)),
absentEntries: make([]string, 0, len(scope)),
}
for index, relativePath := range scope {
archivePath := fmt.Sprintf("/tmp/arcane-workspace/backup-%d.tar", index)
- stdout, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceBackupCreateScriptInternal, "sh", relativePath, archivePath})
+ stdout, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"sh", "-c", volumeWorkspaceBackupCreateScript, "sh", relativePath, archivePath})
if err != nil {
- return nil, classifyVolumeWorkspaceExecErrorInternal(err, stderr, "back up volume workspace path")
+ return nil, classifyVolumeWorkspaceExecError(err, stderr, "back up volume workspace path")
}
fields := strings.Split(stdout, "\x00")
switch {
case len(fields) >= 1 && fields[0] == "present":
- backup.archives = append(backup.archives, volumeWorkspaceBackupArchiveInternal{
+ backup.archives = append(backup.archives, volumeWorkspaceBackupArchive{
relativePath: relativePath,
archivePath: archivePath,
})
@@ -1025,11 +1042,11 @@ func (s *Service) backupVolumeWorkspaceScopeInternal(ctx context.Context, contai
return nil, errors.New("invalid volume workspace backup path response")
}
}
- backup.absentEntries = normalizeVolumeWorkspaceScopeInternal(backup.absentEntries)
+ backup.absentEntries = normalizeVolumeWorkspaceScope(backup.absentEntries)
return backup, nil
}
-func volumeWorkspaceRollbackPathsInternal(backup *volumeWorkspaceBackupInternal) []string {
+func volumeWorkspaceRollbackPaths(backup *volumeWorkspaceBackup) []string {
removePaths := slices.Clone(backup.absentEntries)
for _, archive := range backup.archives {
removePaths = append(removePaths, archive.relativePath)
@@ -1045,49 +1062,7 @@ func volumeWorkspaceRollbackPathsInternal(backup *volumeWorkspaceBackupInternal)
return removePaths
}
-func (s *Service) restoreVolumeWorkspaceScopeInternal(ctx context.Context, containerID string, backup *volumeWorkspaceBackupInternal) error {
- removePaths := volumeWorkspaceRollbackPathsInternal(backup)
- args := append([]string{"sh", "-c", `set -e
-for rel do rm -rf -- "/volume/$rel"; done`, "sh"}, removePaths...)
- if _, stderr, err := s.deps.Exec(ctx, containerID, "", args); err != nil {
- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "remove failed volume workspace changes")
- }
-
- for _, archive := range backup.archives {
- parent := path.Dir(archive.relativePath)
- containerParent := "/volume"
- if parent != "." {
- containerParent = path.Join(containerParent, parent)
- }
- if _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"mkdir", "-p", containerParent}); err != nil {
- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "recreate volume workspace backup parent")
- }
- if _, stderr, err := s.deps.Exec(ctx, containerID, "", []string{"tar", "-xf", archive.archivePath, "-C", containerParent}); err != nil {
- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "restore volume workspace backup")
- }
- }
- return nil
-}
-
-func (s *Service) restoreVolumeWorkspaceFileInternal(ctx context.Context, containerID, volumeName, backupID, rel string) error {
- if err := s.validateVolumeWorkspacePathInternal(ctx, containerID, rel, true); err != nil {
- return err
- }
- filename, cleaned, err := s.deps.ArchiveFileTarget(ctx, volumeName, backupID, rel)
- if err != nil {
- return err
- }
- stderr, err := s.deps.RestoreArchiveMembers(ctx, containerID, filename, []string{cleaned})
- if err != nil {
- return classifyVolumeWorkspaceExecErrorInternal(err, stderr, "restore volume workspace file")
- }
- if strings.TrimSpace(stderr) != "" {
- slog.DebugContext(ctx, "volume workspace change stderr", "operation", volume.FileOpRestoreFile, "path", rel, "stderr", strings.TrimSpace(stderr))
- }
- return nil
-}
-
-func normalizeVolumeWorkspaceScopeInternal(paths []string) []string {
+func normalizeVolumeWorkspaceScope(paths []string) []string {
normalized := slices.Clone(paths)
slices.Sort(normalized)
normalized = slices.Compact(normalized)
@@ -1104,7 +1079,7 @@ func normalizeVolumeWorkspaceScopeInternal(paths []string) []string {
return result
}
-func volumeWorkspaceBackupScopeInternal(changes []volume.WorkspaceFileChange) ([]string, error) {
+func volumeWorkspaceBackupScope(changes []volume.WorkspaceFileChange) ([]string, error) {
paths := make([]string, 0, len(changes)*2)
for _, change := range changes {
rel, err := kit.NormalizeRelativePath(change.RelativePath)
@@ -1130,10 +1105,10 @@ func volumeWorkspaceBackupScopeInternal(changes []volume.WorkspaceFileChange) ([
paths = append(paths, path.Join(parent, path.Base(rel)))
}
}
- return normalizeVolumeWorkspaceScopeInternal(paths), nil
+ return normalizeVolumeWorkspaceScope(paths), nil
}
-func validateVolumeWorkspaceFileChangeInternal(change volume.WorkspaceFileChange) error {
+func validateVolumeWorkspaceFileChange(change volume.WorkspaceFileChange) error {
if _, err := kit.NormalizeRelativePath(change.RelativePath); err != nil {
return fmt.Errorf("invalid volume workspace path: %w", err)
}
@@ -1176,14 +1151,7 @@ func validateVolumeWorkspaceFileChangeInternal(change volume.WorkspaceFileChange
return nil
}
-func validateVolumeWorkspaceRevisionInternal(expected, current string) error {
- if strings.TrimSpace(expected) != current {
- return common.Classify(common.ErrVolumeWorkspaceConflict, errors.New("volume workspace changed; refresh it and try again"))
- }
- return nil
-}
-
-func classifyVolumeWorkspaceHelperSupportErrorInternal(err error) error {
+func classifyVolumeWorkspaceHelperSupportError(err error) error {
switch {
case errdefs.IsNotFound(err):
return common.Classify(common.ErrVolumeWorkspaceNotFound, err)
@@ -1194,7 +1162,7 @@ func classifyVolumeWorkspaceHelperSupportErrorInternal(err error) error {
}
}
-func classifyVolumeWorkspaceExecErrorInternal(err error, stderr, fallbackContext string) error {
+func classifyVolumeWorkspaceExecError(err error, stderr, fallbackContext string) error {
var response types.ErrorResponse
if decodeErr := json.Unmarshal([]byte(strings.TrimSpace(stderr)), &response); decodeErr == nil && response.Version == types.ProtocolVersion {
switch response.Code {
@@ -1240,35 +1208,34 @@ func classifyVolumeWorkspaceExecErrorInternal(err error, stderr, fallbackContext
}
}
-func volumeWorkspaceFileContentResponseInternal(relativePath, kind string, size int64, content []byte, maxFileSizeBytes int64) (*workspace.FileContent, error) {
- response := &workspace.FileContent{
- Path: "/" + relativePath,
- RelativePath: relativePath,
- Name: path.Base(relativePath),
- MimeType: http.DetectContentType(content),
- Size: size,
- }
- if kind == "special" {
- response.ReadOnlyReason = workspace.FileReadOnlySpecial
- return response, nil
- }
- if kind != "regular" {
+func volumeWorkspaceFileContentResponse(relativePath, kind string, size int64, content []byte, maxFileSizeBytes int64) (*workspace.FileContent, error) {
+ if kind != "special" && kind != "regular" {
return nil, errors.New("invalid volume workspace file kind")
}
- if size > maxFileSizeBytes {
- response.ReadOnlyReason = workspace.FileReadOnlyTooLarge
- return response, nil
- }
- if !workspacepkg.IsTextContent(content) {
- response.ReadOnlyReason = workspace.FileReadOnlyBinary
- return response, nil
- }
- response.Editable = true
- response.Content = string(content)
- return response, nil
+ readOnlyReason := func() string {
+ switch {
+ case kind == "special":
+ return workspace.FileReadOnlySpecial
+ case size > maxFileSizeBytes:
+ return workspace.FileReadOnlyTooLarge
+ case !workspacepkg.IsTextContent(content):
+ return workspace.FileReadOnlyBinary
+ }
+ return ""
+ }()
+ return &workspace.FileContent{
+ Path: "/" + relativePath,
+ RelativePath: relativePath,
+ Name: path.Base(relativePath),
+ MimeType: http.DetectContentType(content),
+ Size: size,
+ ReadOnlyReason: readOnlyReason,
+ Editable: readOnlyReason == "",
+ Content: kit.Ternary(readOnlyReason == "", string(content), ""),
+ }, nil
}
-func volumeWorkspaceEntryFromACFSInternal(entry types.Entry, maxFileSizeBytes int64) (workspace.FileEntry, string) {
+func volumeWorkspaceEntryFromACFS(entry types.Entry, maxFileSizeBytes int64) (workspace.FileEntry, string) {
fileEntry := acfsutils.FileEntry(entry)
fileEntry.RelativePath = strings.TrimPrefix(entry.Path, "/")
switch {
@@ -1292,7 +1259,7 @@ func volumeWorkspaceEntryFromACFSInternal(entry types.Entry, maxFileSizeBytes in
}
}
-func decodeVolumeWorkspaceWalkInternal(source io.Reader, maxEntries int, maxFileSizeBytes int64) (*workspace.Workspace, error) {
+func decodeVolumeWorkspaceWalk(source io.Reader, maxEntries int, maxFileSizeBytes int64) (*workspace.Workspace, error) {
decoder := jsontext.NewDecoder(source)
files := make([]workspace.FileEntry, 0, min(maxEntries, 256))
classifications := make(map[string]string, min(maxEntries, 256))
@@ -1327,7 +1294,7 @@ func decodeVolumeWorkspaceWalkInternal(source io.Reader, maxEntries int, maxFile
if len(files) >= maxEntries {
return nil, errors.New("acfs walk exceeded the requested entry limit")
}
- fileEntry, classification := volumeWorkspaceEntryFromACFSInternal(*record.Entry, maxFileSizeBytes)
+ fileEntry, classification := volumeWorkspaceEntryFromACFS(*record.Entry, maxFileSizeBytes)
classifications[fileEntry.RelativePath] = classification
files = append(files, fileEntry)
}
diff --git a/backend/internal/volume/children/workspace/service_test.go b/backend/internal/volume/children/workspace/service_test.go
index d23b048a3..ebebe0b9b 100644
--- a/backend/internal/volume/children/workspace/service_test.go
+++ b/backend/internal/volume/children/workspace/service_test.go
@@ -31,7 +31,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
)
-func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
+func newVolumeWorkspaceTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
dockerClient, err := client.New(
client.WithHost(server.URL),
@@ -43,7 +43,7 @@ func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.S
return dockerClient
}
-func TestValidateVolumeHelperSupportInternal(t *testing.T) {
+func TestValidateVolumeHelperSupport(t *testing.T) {
var requests []string
var options map[string]string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -59,18 +59,18 @@ func TestValidateVolumeHelperSupportInternal(t *testing.T) {
}))
t.Cleanup(server.Close)
- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClient(t, server))
service := NewService(Dependencies{Docker: dockerService})
- require.NoError(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))
+ require.NoError(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))
require.Equal(t, []string{"GET /v1.41/volumes/workspace-volume"}, requests)
options = map[string]string{"type": "nfs"}
- require.NoError(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))
+ require.NoError(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))
options = map[string]string{"type": "none"}
- require.Error(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))
+ require.Error(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))
options = map[string]string{"o": "bind,rw"}
- require.Error(t, service.validateVolumeHelperSupportInternal(t.Context(), "workspace-volume"))
+ require.Error(t, service.validateVolumeHelperSupport(t.Context(), "workspace-volume"))
}
func TestUpdateVolumeWorkspaceValidationFailureReturnsNoWorkspace(t *testing.T) {
@@ -86,33 +86,33 @@ func TestUpdateVolumeWorkspaceValidationFailureReturnsNoWorkspace(t *testing.T)
require.ErrorIs(t, err, common.ErrVolumeWorkspaceBadRequest)
}
-func TestVolumeWorkspaceFileContentResponseInternal(t *testing.T) {
+func TestVolumeWorkspaceFileContentResponse(t *testing.T) {
const maxFileSizeBytes int64 = 10 * 1024 * 1024
text := []byte("hello")
- below, err := volumeWorkspaceFileContentResponseInternal("notes.txt", "regular", maxFileSizeBytes-1, text, maxFileSizeBytes)
+ below, err := volumeWorkspaceFileContentResponse("notes.txt", "regular", maxFileSizeBytes-1, text, maxFileSizeBytes)
require.NoError(t, err)
require.True(t, below.Editable)
require.Equal(t, "hello", below.Content)
- atLimit, err := volumeWorkspaceFileContentResponseInternal("notes.txt", "regular", maxFileSizeBytes, text, maxFileSizeBytes)
+ atLimit, err := volumeWorkspaceFileContentResponse("notes.txt", "regular", maxFileSizeBytes, text, maxFileSizeBytes)
require.NoError(t, err)
require.True(t, atLimit.Editable)
- above, err := volumeWorkspaceFileContentResponseInternal("notes.txt", "regular", maxFileSizeBytes+1, text, maxFileSizeBytes)
+ above, err := volumeWorkspaceFileContentResponse("notes.txt", "regular", maxFileSizeBytes+1, text, maxFileSizeBytes)
require.NoError(t, err)
require.False(t, above.Editable)
require.Equal(t, workspace.FileReadOnlyTooLarge, above.ReadOnlyReason)
- binaryContent, err := volumeWorkspaceFileContentResponseInternal("data.bin", "regular", 2, []byte{0xff, 0x00}, maxFileSizeBytes)
+ binaryContent, err := volumeWorkspaceFileContentResponse("data.bin", "regular", 2, []byte{0xff, 0x00}, maxFileSizeBytes)
require.NoError(t, err)
require.Equal(t, workspace.FileReadOnlyBinary, binaryContent.ReadOnlyReason)
- special, err := volumeWorkspaceFileContentResponseInternal("pipe", "special", 0, nil, maxFileSizeBytes)
+ special, err := volumeWorkspaceFileContentResponse("pipe", "special", 0, nil, maxFileSizeBytes)
require.NoError(t, err)
require.Equal(t, workspace.FileReadOnlySpecial, special.ReadOnlyReason)
}
-func TestValidateVolumeWorkspaceFileChangeInternalOperationsAndMultipartMapping(t *testing.T) {
+func TestValidateVolumeWorkspaceFileChangeOperationsAndMultipartMapping(t *testing.T) {
first := 0
second := 1
tests := []volumetypes.WorkspaceFileChange{
@@ -126,36 +126,36 @@ func TestValidateVolumeWorkspaceFileChangeInternalOperationsAndMultipartMapping(
{Operation: volumetypes.FileOpRestoreFile, RelativePath: "restored.txt", BackupID: "backup-id"},
}
for _, change := range tests {
- require.NoError(t, validateVolumeWorkspaceFileChangeInternal(change), change.Operation)
+ require.NoError(t, validateVolumeWorkspaceFileChange(change), change.Operation)
}
- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{
+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{
Operation: volumetypes.FileOpUpdateFile, RelativePath: "missing.bin",
}))
- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{
+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{
Operation: volumetypes.FileOpRestoreFile, RelativePath: "file.txt",
}))
- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{
+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{
Operation: "unknown", RelativePath: "file.txt",
}))
- require.Error(t, validateVolumeWorkspaceFileChangeInternal(volumetypes.WorkspaceFileChange{
+ require.Error(t, validateVolumeWorkspaceFileChange(volumetypes.WorkspaceFileChange{
Operation: volumetypes.FileOpDelete, RelativePath: "../escape",
}))
}
-func TestVolumeWorkspaceBackupScopeInternalCoversSourcesAndDestinations(t *testing.T) {
+func TestVolumeWorkspaceBackupScopeCoversSourcesAndDestinations(t *testing.T) {
changes := []volumetypes.WorkspaceFileChange{
{Operation: volumetypes.FileOpRename, RelativePath: "docs/a.txt", NewName: "b.txt"},
{Operation: volumetypes.FileOpMove, RelativePath: "cache", NewParentPath: "docs"},
{Operation: volumetypes.FileOpCreateFile, RelativePath: "docs/a.txt/child", UploadIndex: new(0)},
}
- scope, err := volumeWorkspaceBackupScopeInternal(changes)
+ scope, err := volumeWorkspaceBackupScope(changes)
require.NoError(t, err)
require.Equal(t, []string{"cache", "docs/a.txt", "docs/b.txt", "docs/cache"}, scope)
}
-func TestNormalizeVolumeWorkspaceScopeInternalCollapsesAbsentAncestors(t *testing.T) {
- require.Equal(t, []string{"cache", "docs"}, normalizeVolumeWorkspaceScopeInternal([]string{
+func TestNormalizeVolumeWorkspaceScopeCollapsesAbsentAncestors(t *testing.T) {
+ require.Equal(t, []string{"cache", "docs"}, normalizeVolumeWorkspaceScope([]string{
"docs/b.txt",
"docs",
"cache/item",
@@ -165,21 +165,21 @@ func TestNormalizeVolumeWorkspaceScopeInternalCollapsesAbsentAncestors(t *testin
}))
}
-func TestVolumeWorkspaceRollbackPathsInternalRemovesDeepestFirst(t *testing.T) {
- backup := &volumeWorkspaceBackupInternal{
- archives: []volumeWorkspaceBackupArchiveInternal{
+func TestVolumeWorkspaceRollbackPathsRemovesDeepestFirst(t *testing.T) {
+ backup := &volumeWorkspaceBackup{
+ archives: []volumeWorkspaceBackupArchive{
{relativePath: "docs/file.txt", archivePath: "/tmp/file.tar"},
{relativePath: "cache", archivePath: "/tmp/cache.tar"},
},
absentEntries: []string{"docs/new/deep", "other"},
}
- require.Equal(t, []string{"docs/new/deep", "docs/file.txt", "other", "cache"}, volumeWorkspaceRollbackPathsInternal(backup))
+ require.Equal(t, []string{"docs/new/deep", "docs/file.txt", "other", "cache"}, volumeWorkspaceRollbackPaths(backup))
}
func TestVolumeWorkspaceHelperScriptsUseSupportedTooling(t *testing.T) {
helperScripts := []string{
- volumeWorkspaceValidatePathScriptInternal,
- volumeWorkspaceBackupCreateScriptInternal,
+ volumeWorkspaceValidatePathScript,
+ volumeWorkspaceBackupCreateScript,
}
scripts := strings.Join(helperScripts, "\n")
for _, unsupported := range []string{
@@ -201,9 +201,9 @@ func TestVolumeWorkspaceHelperScriptsUseSupportedTooling(t *testing.T) {
} {
require.NotContains(t, scripts, unsupported)
}
- require.Contains(t, volumeWorkspaceBackupCreateScriptInternal, `printf 'absent\0%s\0'`)
- require.Contains(t, volumeWorkspaceBackupCreateScriptInternal, `cd "$parent"`)
- require.NotContains(t, volumeWorkspaceBackupCreateScriptInternal, " -C ")
+ require.Contains(t, volumeWorkspaceBackupCreateScript, `printf 'absent\0%s\0'`)
+ require.Contains(t, volumeWorkspaceBackupCreateScript, `cd "$parent"`)
+ require.NotContains(t, volumeWorkspaceBackupCreateScript, " -C ")
if shellPath, err := exec.LookPath("sh"); err == nil {
for index, script := range helperScripts {
output, syntaxErr := exec.Command(shellPath, "-n", "-c", script).CombinedOutput()
@@ -212,24 +212,19 @@ func TestVolumeWorkspaceHelperScriptsUseSupportedTooling(t *testing.T) {
}
}
-func TestClassifyVolumeWorkspaceExecErrorInternal(t *testing.T) {
+func TestClassifyVolumeWorkspaceExecError(t *testing.T) {
base := errors.New("exit status 1")
- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "ARCANE_SYMLINK", "execute"), common.ErrVolumeWorkspaceForbidden)
- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "ARCANE_NOT_FOUND", "execute"), common.ErrVolumeWorkspaceNotFound)
- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "ARCANE_COLLISION", "execute"), common.ErrVolumeWorkspaceBadRequest)
- require.ErrorIs(t, classifyVolumeWorkspaceExecErrorInternal(base, "unexpected", "execute"), base)
+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "ARCANE_SYMLINK", "execute"), common.ErrVolumeWorkspaceForbidden)
+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "ARCANE_NOT_FOUND", "execute"), common.ErrVolumeWorkspaceNotFound)
+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "ARCANE_COLLISION", "execute"), common.ErrVolumeWorkspaceBadRequest)
+ require.ErrorIs(t, classifyVolumeWorkspaceExecError(base, "unexpected", "execute"), base)
}
-func TestClassifyVolumeWorkspaceHelperSupportErrorInternal(t *testing.T) {
- require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportErrorInternal(errdefs.ErrNotFound), common.ErrVolumeWorkspaceNotFound)
- require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportErrorInternal(errors.New("volume uses a custom mount configuration")), common.ErrVolumeWorkspaceBadRequest)
+func TestClassifyVolumeWorkspaceHelperSupportError(t *testing.T) {
+ require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportError(errdefs.ErrNotFound), common.ErrVolumeWorkspaceNotFound)
+ require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportError(errors.New("volume uses a custom mount configuration")), common.ErrVolumeWorkspaceBadRequest)
unexpected := errors.New("docker unavailable")
- require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportErrorInternal(unexpected), unexpected)
-}
-
-func TestValidateVolumeWorkspaceRevisionInternal(t *testing.T) {
- require.NoError(t, validateVolumeWorkspaceRevisionInternal(" revision ", "revision"))
- require.ErrorIs(t, validateVolumeWorkspaceRevisionInternal("stale", "current"), common.ErrVolumeWorkspaceConflict)
+ require.ErrorIs(t, classifyVolumeWorkspaceHelperSupportError(unexpected), unexpected)
}
func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testing.T) {
@@ -267,7 +262,7 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi
}))
t.Cleanup(server.Close)
- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)
+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)
service := NewService(Dependencies{
Docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),
Exec: func(_ context.Context, containerID, _ string, cmd []string) (string, string, error) {
@@ -278,16 +273,16 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi
})
firstUpload := 0
secondUpload := 1
- firstStaged, err := service.stageVolumeWorkspaceChangesInternal(t.Context(), dockerClient, "helper", []volumetypes.WorkspaceFileChange{
+ firstStaged, err := service.stageVolumeWorkspaceChanges(t.Context(), dockerClient, "helper", []volumetypes.WorkspaceFileChange{
{UploadIndex: &firstUpload},
{},
{UploadIndex: &secondUpload},
- }, map[int][]byte{0: []byte("alpha"), 1: {}}, volumeWorkspaceWriteIdentityInternal{})
+ }, map[int][]byte{0: []byte("alpha"), 1: {}}, volumeWorkspaceWriteIdentity{})
require.NoError(t, err)
- require.Equal(t, volumeWorkspaceStagedFileInternal{path: "/tmp/arcane-workspace/change-0", size: 5}, firstStaged[0])
- require.Equal(t, volumeWorkspaceStagedFileInternal{path: "/tmp/arcane-workspace/change-2", size: 0}, firstStaged[2])
+ require.Equal(t, volumeWorkspaceStagedFile{path: "/tmp/arcane-workspace/change-0", size: 5}, firstStaged[0])
+ require.Equal(t, volumeWorkspaceStagedFile{path: "/tmp/arcane-workspace/change-2", size: 0}, firstStaged[2])
- secondStaged, err := service.stageVolumeWorkspaceChangesInternal(
+ secondStaged, err := service.stageVolumeWorkspaceChanges(
t.Context(),
dockerClient,
"helper",
@@ -301,10 +296,10 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi
"second",
),
},
- volumeWorkspaceWriteIdentityInternal{},
+ volumeWorkspaceWriteIdentity{},
)
require.NoError(t, err)
- require.Equal(t, volumeWorkspaceStagedFileInternal{path: "/tmp/arcane-workspace/change-0", size: 6}, secondStaged[0])
+ require.Equal(t, volumeWorkspaceStagedFile{path: "/tmp/arcane-workspace/change-0", size: 6}, secondStaged[0])
require.Equal(t, [][]string{
{"sh", "-c", "rm -rf -- /tmp/arcane-workspace && mkdir -p -- /tmp/arcane-workspace"},
@@ -316,22 +311,22 @@ func TestStageVolumeWorkspaceChangesClearsAndCopiesContentsInOneArchive(t *testi
}, copiedArchives)
}
-func TestVolumeWorkspaceWriteIdentityFromConfigUserInternal(t *testing.T) {
- root := volumeWorkspaceWriteIdentityInternal{}
+func TestVolumeWorkspaceWriteIdentityFromConfigUser(t *testing.T) {
+ root := volumeWorkspaceWriteIdentity{}
for _, configUser := range []string{"", "root", "0", "0:0", "0:1000", "node", "1000:node", "99999999999999999999"} {
- require.Equalf(t, root, volumeWorkspaceWriteIdentityFromConfigUserInternal(configUser), "config user %q", configUser)
+ require.Equalf(t, root, volumeWorkspaceWriteIdentityFromConfigUser(configUser), "config user %q", configUser)
}
- require.Equal(t, volumeWorkspaceWriteIdentityInternal{execUser: "1000", uid: 1000}, volumeWorkspaceWriteIdentityFromConfigUserInternal("1000"))
- require.Equal(t, volumeWorkspaceWriteIdentityInternal{execUser: "1000:1000", uid: 1000, gid: 1000}, volumeWorkspaceWriteIdentityFromConfigUserInternal(" 1000:1000 "))
- require.Equal(t, volumeWorkspaceWriteIdentityInternal{execUser: "1000:0", uid: 1000, gid: 0}, volumeWorkspaceWriteIdentityFromConfigUserInternal("1000:0"))
+ require.Equal(t, volumeWorkspaceWriteIdentity{execUser: "1000", uid: 1000}, volumeWorkspaceWriteIdentityFromConfigUser("1000"))
+ require.Equal(t, volumeWorkspaceWriteIdentity{execUser: "1000:1000", uid: 1000, gid: 1000}, volumeWorkspaceWriteIdentityFromConfigUser(" 1000:1000 "))
+ require.Equal(t, volumeWorkspaceWriteIdentity{execUser: "1000:0", uid: 1000, gid: 0}, volumeWorkspaceWriteIdentityFromConfigUser("1000:0"))
}
func TestVolumeWorkspaceWritesUseRuntimeIdentity(t *testing.T) {
- type execRequestInternal struct {
+ type execRequest struct {
user string
cmd []string
}
- var execRequests []execRequestInternal
+ var execRequests []execRequest
var archiveQueries []string
var archiveHeaders []*tar.Header
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -361,22 +356,22 @@ func TestVolumeWorkspaceWritesUseRuntimeIdentity(t *testing.T) {
}))
t.Cleanup(server.Close)
- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)
+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)
applyResponse, err := json.Marshal(types.ApplyResponse{Version: types.ProtocolVersion, Applied: 1})
require.NoError(t, err)
service := NewService(Dependencies{
Docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),
Exec: func(_ context.Context, _, execUser string, cmd []string) (string, string, error) {
- execRequests = append(execRequests, execRequestInternal{user: execUser, cmd: cmd})
+ execRequests = append(execRequests, execRequest{user: execUser, cmd: cmd})
return string(applyResponse), "", nil
},
})
- identity := volumeWorkspaceWriteIdentityFromConfigUserInternal("1000:1000")
+ identity := volumeWorkspaceWriteIdentityFromConfigUser("1000:1000")
uploadIndex := 0
changes := []volumetypes.WorkspaceFileChange{{Operation: volumetypes.FileOpCreateFile, RelativePath: "config.txt", UploadIndex: &uploadIndex}}
- stagedFiles, err := service.stageVolumeWorkspaceChangesInternal(t.Context(), dockerClient, "helper", changes, map[int][]byte{0: []byte("content")}, identity)
+ stagedFiles, err := service.stageVolumeWorkspaceChanges(t.Context(), dockerClient, "helper", changes, map[int][]byte{0: []byte("content")}, identity)
require.NoError(t, err)
- require.NoError(t, service.executeVolumeWorkspaceACFSBatchInternal(t.Context(), dockerClient, "helper", changes, stagedFiles, 0, identity))
+ require.NoError(t, service.executeVolumeWorkspaceACFSBatch(t.Context(), dockerClient, "helper", changes, stagedFiles, 0, identity))
require.Len(t, execRequests, 2)
require.Empty(t, execRequests[0].user)
@@ -417,7 +412,7 @@ func TestCreateVolumeWorkspaceMutationContainerUsesDedicatedBackupHelper(t *test
}))
t.Cleanup(server.Close)
- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)
+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)
acquireCalls := 0
service := NewService(Dependencies{
Docker: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),
@@ -425,7 +420,7 @@ func TestCreateVolumeWorkspaceMutationContainerUsesDedicatedBackupHelper(t *test
acquireCalls++
return "cached-helper", func() {}, nil
},
- HelperImage: func(context.Context, *client.Client) (string, error) {
+ HelperImage: func(context.Context) (string, error) {
return volumehelper.ToolsImage(""), nil
},
BackupStorageMount: func(_ context.Context, _ *client.Client, target string, readOnly bool) (mount.Mount, error) {
@@ -433,7 +428,7 @@ func TestCreateVolumeWorkspaceMutationContainerUsesDedicatedBackupHelper(t *test
},
})
- containerID, cleanup, err := service.createVolumeWorkspaceMutationContainerInternal(t.Context(), "workspace-volume", true)
+ containerID, cleanup, err := service.createVolumeWorkspaceMutationContainer(t.Context(), "workspace-volume", true)
require.NoError(t, err)
require.Equal(t, "restore-helper", containerID)
require.Equal(t, 1, createCalls)
@@ -508,7 +503,7 @@ acfs apply --root /volume --staging /tmp/staging --manifest manifest.json >/dev/
sh -c "$1" sh dest/b.txt /tmp/backup.tar >/dev/null
acfs remove --root /volume --path /dest/b.txt
tar -xf /tmp/backup.tar -C /volume/dest
-head -c 7 /volume/dest/b.txt`, volumeWorkspaceBackupCreateScriptInternal)
+head -c 7 /volume/dest/b.txt`, volumeWorkspaceBackupCreateScript)
require.Equal(t, "updated", restored)
runInVolume(volumeName, `mkdir -p "/volume/Test Folder"
@@ -517,16 +512,16 @@ printf spaced > "/volume/Test Folder/file.txt"`)
sh -c "$1" sh "Test Folder" /tmp/space-backup.tar >/dev/null
rm -rf "/volume/Test Folder"
tar -xf /tmp/space-backup.tar -C /volume
-head -c 6 "/volume/Test Folder/file.txt"`, volumeWorkspaceBackupCreateScriptInternal)
+head -c 6 "/volume/Test Folder/file.txt"`, volumeWorkspaceBackupCreateScript)
require.Equal(t, "spaced", spacedRestored)
missing := runInVolume(volumeName, `set -e
sh -c "$1" sh test.txt /tmp/missing-backup.tar
-if stat -c '%A' -- /tmp/missing-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScriptInternal)
+if stat -c '%A' -- /tmp/missing-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScript)
require.Equal(t, "absent\x00test.txt\x00", missing)
missingNested := runInVolume(volumeName, `set -e
sh -c "$1" sh "New Folder/test.txt" /tmp/missing-nested-backup.tar
-if stat -c '%A' -- /tmp/missing-nested-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScriptInternal)
+if stat -c '%A' -- /tmp/missing-nested-backup.tar >/dev/null 2>&1; then exit 1; fi`, volumeWorkspaceBackupCreateScript)
require.Equal(t, "absent\x00New Folder\x00", missingNested)
}
diff --git a/backend/internal/volume/helpers.go b/backend/internal/volume/helpers.go
index b6b77dea1..02303065d 100644
--- a/backend/internal/volume/helpers.go
+++ b/backend/internal/volume/helpers.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"log/slog"
+ "slices"
"strings"
"time"
@@ -27,10 +28,8 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
)
-// volumeHelper tracks a reused helper container and the last
-// time it serviced a request, so idle helpers can be reaped. inUse counts the
-// requests currently holding the helper: the reaper must not remove a helper
-// mid-download just because it was acquired longer ago than the idle timeout.
+// volumeHelper tracks a reused helper container and its last use for idle reaping.
+// inUse counts active holds so the reaper never removes a helper mid-request.
type volumeHelper struct {
id string
lastUsedAt time.Time
@@ -38,7 +37,7 @@ type volumeHelper struct {
protocol int
}
-func (s *VolumeService) requireVolumeHelperACFSInternal(ctx context.Context, volumeName, containerID string) error {
+func (s *VolumeService) requireVolumeHelperACFS(ctx context.Context, volumeName, containerID string) error {
s.helperMu.Lock()
helper := s.helperByVolume[volumeName]
if helper != nil && helper.id == containerID && helper.protocol >= types.ProtocolVersion {
@@ -47,7 +46,7 @@ func (s *VolumeService) requireVolumeHelperACFSInternal(ctx context.Context, vol
}
s.helperMu.Unlock()
- stdout, stderr, err := s.execInContainerInternal(ctx, containerID, "", []string{"acfs", "version"})
+ stdout, stderr, err := s.execInContainer(ctx, containerID, "", []string{"acfs", "version"})
if err != nil {
return fmt.Errorf("%s: %w", "volume workspace requires an ACFS-capable tools image: "+strings.TrimSpace(stderr), err)
}
@@ -67,7 +66,7 @@ func (s *VolumeService) requireVolumeHelperACFSInternal(ctx context.Context, vol
return nil
}
-func (s *VolumeService) toolsImageInternal() string {
+func (s *VolumeService) toolsImage() string {
registry := ""
if s.settingsService != nil {
registry = s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value
@@ -75,52 +74,31 @@ func (s *VolumeService) toolsImageInternal() string {
return volumehelper.ToolsImage(registry)
}
-func isUnlabeledVolumeHelperContainerInternal(c container.Summary) bool {
+func isUnlabeledVolumeHelperContainer(c container.Summary) bool {
if internal, _ := kit.ParseBool(c.Labels[libarcane.InternalResourceLabel]); !internal {
return false
}
command := strings.ToLower(c.Command)
- if !strings.Contains(command, "sleep") || !strings.Contains(command, "infinity") {
- return false
- }
-
- for _, mounted := range c.Mounts {
- if mounted.Destination == "/volume" {
- return true
- }
- }
-
- return false
+ return strings.Contains(command, "sleep") && strings.Contains(command, "infinity") && slices.ContainsFunc(c.Mounts, func(mounted container.MountPoint) bool {
+ return mounted.Destination == "/volume"
+ })
}
-func isVolumeHelperContainerInternal(c container.Summary) bool {
- if isUnlabeledVolumeHelperContainerInternal(c) {
- return true
- }
+func isVolumeHelperContainer(c container.Summary) bool {
if internal, _ := kit.ParseBool(c.Labels[libarcane.InternalResourceLabel]); !internal {
return false
}
-
- return strings.EqualFold(c.Labels[volumehelper.ContainerLabel], "true")
+ return strings.EqualFold(c.Labels[volumehelper.ContainerLabel], "true") || isUnlabeledVolumeHelperContainer(c)
}
-func (s *VolumeService) getVolumeHelperImageInternal(ctx context.Context, dockerClient *client.Client) (string, error) {
+func (s *VolumeService) getVolumeHelperImage(ctx context.Context) (string, error) {
slog.DebugContext(ctx, "volume service: resolve helper image")
- var err error
- if dockerClient == nil {
- if s.dockerService == nil {
- return "", errors.New("docker service unavailable")
- }
- dockerClient, err = s.dockerService.GetClient(ctx)
- if err != nil {
- return "", fmt.Errorf("failed to get docker client: %w", err)
- }
- }
-
- toolsImage := s.toolsImageInternal()
+ toolsImage := s.toolsImage()
- if _, imageInspectErr := dockerClient.ImageInspect(ctx, toolsImage); imageInspectErr == nil {
+ if _, imageInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ImageInspectResult, error) {
+ return dockerClient.ImageInspect(ctx, toolsImage)
+ }); imageInspectErr == nil {
slog.InfoContext(ctx, "volume service: helper image strategy selected", "strategy", "tools-local", "image", toolsImage)
return toolsImage, nil
}
@@ -139,6 +117,10 @@ func (s *VolumeService) getVolumeHelperImageInternal(ctx context.Context, docker
slog.WarnContext(ctx, "volume service: image service unavailable, attempting arcane fallback")
}
+ dockerClient, err := s.dockerService.GetClient(ctx)
+ if err != nil {
+ return "", fmt.Errorf("failed to get docker client: %w", err)
+ }
if fallback, ok := volumehelper.ResolveArcaneRuntimeImage(ctx, dockerClient).Get(); ok {
slog.InfoContext(ctx, "volume service: helper image strategy selected", "strategy", "arcane-fallback", "source", fallback.Source, "image", fallback.Image)
return fallback.Image, nil
@@ -147,37 +129,27 @@ func (s *VolumeService) getVolumeHelperImageInternal(ctx context.Context, docker
return "", fmt.Errorf("failed to resolve helper image: tools image unavailable and arcane fallback not found: %w", pullErr)
}
-func (s *VolumeService) acquireVolumeHelperInternal(ctx context.Context, volumeName string) (string, func(), error) {
+func (s *VolumeService) acquireVolumeHelper(ctx context.Context, volumeName string) (string, func(), error) {
slog.DebugContext(ctx, "volume service: acquire helper container", "volume", volumeName)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return "", nil, err
- }
- // Helpers are shared and reaped when idle; the caller's cleanup
- // releases its in-use hold instead of removing the container. The resolve →
- // acquire gap is racy against the reaper, so retry a resolve whose helper
- // was reaped before this caller could take its hold.
+ // The returned cleanup releases a hold on the shared helper; retry when the reaper wins the resolve-to-acquire race.
for range 3 {
- containerID, resolveHelperErr := s.resolveHelperInternal(ctx, dockerClient, volumeName)
+ containerID, resolveHelperErr := s.resolveHelper(ctx, volumeName)
if resolveHelperErr != nil {
return "", nil, resolveHelperErr
}
- if release, ok := s.acquireHelperInternal(volumeName, containerID); ok {
+ if release, ok := s.acquireHelper(volumeName, containerID); ok {
return containerID, release, nil
}
}
return "", nil, errors.New("failed to acquire volume helper container")
}
-// resolveHelperInternal returns the shared helper container ID
-// for volumeName, creating it if needed. singleflight collapses concurrent
-// misses: without it both requests create a helper and the loser is orphaned,
-// holding the volume mount with no cleanup path until Arcane restarts. The
-// creation itself can pull an image, so it must not run under helperMu.
-func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (string, error) {
+// resolveHelper returns volumeName's shared helper, creating it once via singleflight so concurrent misses never orphan one.
+// Creation can pull an image, so it runs outside helperMu.
+func (s *VolumeService) resolveHelper(ctx context.Context, volumeName string) (string, error) {
resultCh := s.helperGroup.DoChan(volumeName, func() (any, error) {
- helper, err := s.getReusableHelperInternal(ctx, dockerClient, volumeName)
+ helper, err := s.getReusableHelper(ctx, volumeName)
if err != nil {
return nil, err
}
@@ -185,15 +157,11 @@ func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient
return containerID, nil
}
- // Detached from the caller's context so a client that walks away
- // mid-create doesn't abort the helper the other waiters are blocked on,
- // but still bounded: WithoutCancel alone would let a hung daemon or
- // image pull wedge this singleflight key forever, queueing every later
- // workspace request for the volume behind it with no recovery until restart.
+ // Detached so one caller leaving does not abort the shared create, but bounded so a hung pull cannot wedge the key.
createCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerImagePull)
defer cancel()
- containerID, _, createErr := s.startHelperContainerInternal(createCtx, dockerClient, volumeName)
+ containerID, _, createErr := s.startHelperContainer(createCtx, volumeName)
if createErr != nil {
return nil, createErr
}
@@ -204,9 +172,7 @@ func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient
return containerID, nil
})
- // DoChan instead of Do so a caller whose request is canceled unblocks
- // immediately; the shared create keeps running for the remaining waiters
- // (and lands in helperByVolume for the next request even if none remain).
+ // DoChan lets a canceled caller return at once while the shared create finishes for the others.
var shared any
select {
case <-ctx.Done():
@@ -225,13 +191,9 @@ func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient
return containerID, nil
}
-// acquireHelperInternal takes an in-use hold on volumeName's helper, reporting
-// false when the helper has been removed (e.g. reaped) since it was resolved.
-// The returned release drops the hold and refreshes the idle clock, so the idle
-// timeout is measured from the end of the last request, not its start — a
-// download running longer than the timeout used to have its helper reaped out
-// from under it, truncating the stream.
-func (s *VolumeService) acquireHelperInternal(volumeName, containerID string) (func(), bool) {
+// acquireHelper takes a hold on volumeName's helper, or reports false if it was removed since being resolved.
+// Release refreshes the idle clock so the timeout counts from the end of the last request.
+func (s *VolumeService) acquireHelper(volumeName, containerID string) (func(), bool) {
s.helperMu.Lock()
defer s.helperMu.Unlock()
@@ -253,11 +215,9 @@ func (s *VolumeService) acquireHelperInternal(volumeName, containerID string) (f
}, true
}
-// startHelperContainerInternal creates and starts a volume helper container and
-// returns a cleanup that removes it. Tracking of reusable helpers is
-// the caller's responsibility.
-func (s *VolumeService) startHelperContainerInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (string, func(), error) {
- helperImage, err := s.getVolumeHelperImageInternal(ctx, dockerClient)
+// startHelperContainer creates and starts a helper container and returns a cleanup that removes it.
+func (s *VolumeService) startHelperContainer(ctx context.Context, volumeName string) (string, func(), error) {
+ helperImage, err := s.getVolumeHelperImage(ctx)
if err != nil {
return "", nil, err
}
@@ -271,32 +231,38 @@ func (s *VolumeService) startHelperContainerInternal(ctx context.Context, docker
hostConfig := volumehelper.HostConfig(helperImage, []string{volumeName + ":/volume"}, nil)
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: config,
- HostConfig: hostConfig,
+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerCreateResult, error) {
+ return dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: config,
+ HostConfig: hostConfig,
+ })
})
if err != nil {
return "", nil, fmt.Errorf("failed to create temp container: %w", err)
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerStartResult, error) {
+ return dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
startErr := fmt.Errorf("failed to start temp container: %w", containerStartErr)
- cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)
- defer cancel()
- if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {
+ if _, cleanupErr := s.dockerService.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ }); cleanupErr != nil && !errdefs.IsNotFound(cleanupErr) {
return "", nil, errors.Join(startErr, fmt.Errorf("failed to remove temp container after start failure: %w", cleanupErr))
}
return "", nil, startErr
}
cleanup := func() {
- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
+ _, _ = s.dockerService.Call(context.WithoutCancel(ctx), func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
+ })
}
return resp.ID, cleanup, nil
}
-func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (mo.Option[string], error) {
+func (s *VolumeService) getReusableHelper(ctx context.Context, volumeName string) (mo.Option[string], error) {
s.helperMu.Lock()
helper := s.helperByVolume[volumeName]
s.helperMu.Unlock()
@@ -304,19 +270,22 @@ func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerCli
return mo.None[string](), nil
}
- inspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, helper.id, client.ContainerInspectOptions{})
+ inspect, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerInspectResult, error) {
+ return compat.ContainerInspectWithCompatibility(ctx, dockerClient, helper.id, client.ContainerInspectOptions{})
+ })
if err != nil && !errdefs.IsNotFound(err) {
return mo.None[string](), fmt.Errorf("failed to inspect volume helper container: %w", err)
}
- if err == nil && (inspect.Container.State == nil || !inspect.Container.State.Running) {
- removeCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI)
- _, removeErr := dockerClient.ContainerRemove(removeCtx, helper.id, volumehelper.RemoveOptions())
- cancel()
+ running := err == nil && kit.FromPtr(inspect.Container.State).Running
+ if err == nil && !running {
+ _, removeErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, helper.id, volumehelper.RemoveOptions())
+ })
if removeErr != nil && !errdefs.IsNotFound(removeErr) {
return mo.None[string](), fmt.Errorf("failed to remove stopped volume helper container: %w", removeErr)
}
}
- if err != nil || inspect.Container.State == nil || !inspect.Container.State.Running {
+ if !running {
s.helperMu.Lock()
if s.helperByVolume[volumeName] == helper {
delete(s.helperByVolume, volumeName)
@@ -325,14 +294,13 @@ func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerCli
return mo.None[string](), nil
}
- s.touchHelperInternal(volumeName)
+ s.touchHelper(volumeName)
return mo.Some(helper.id), nil
}
-// touchHelperInternal records that the helper for volumeName just serviced a
-// request, resetting its idle clock. No-op if the entry is gone.
-func (s *VolumeService) touchHelperInternal(volumeName string) {
+// touchHelper resets the idle clock of volumeName's helper, if any.
+func (s *VolumeService) touchHelper(volumeName string) {
s.helperMu.Lock()
defer s.helperMu.Unlock()
if helper := s.helperByVolume[volumeName]; helper != nil {
@@ -341,12 +309,6 @@ func (s *VolumeService) touchHelperInternal(volumeName string) {
}
func (s *VolumeService) CleanupHelperContainers(ctx context.Context) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- slog.WarnContext(ctx, "failed to get docker client for helper cleanup", "error", err)
- return
- }
-
s.helperMu.Lock()
helperIDs := make([]string, 0, len(s.helperByVolume))
for _, helper := range s.helperByVolume {
@@ -358,32 +320,27 @@ func (s *VolumeService) CleanupHelperContainers(ctx context.Context) {
s.helperMu.Unlock()
for _, containerID := range helperIDs {
- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions()); containerRemoveErr != nil {
+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions())
+ }); containerRemoveErr != nil {
slog.WarnContext(ctx, "failed to remove helper container", "containerId", containerID, "error", containerRemoveErr.Error())
}
}
}
-// ReapIdleHelpers removes reused helper containers that have
-// not serviced a request within idleTimeout. It is map-driven (orphaned helpers
-// not tracked in helperByVolume are left to the startup orphan sweep). Entries
-// are removed from the map before the container is removed, so a concurrent
-// request simply gets a cache miss and re-creates a fresh helper.
+// ReapIdleHelpers removes tracked helpers idle for idleTimeout; untracked orphans are left to the startup sweep.
func (s *VolumeService) ReapIdleHelpers(ctx context.Context, idleTimeout time.Duration) (int, error) {
if idleTimeout <= 0 {
return 0, nil
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return 0, fmt.Errorf("failed to get docker client for idle helper reap: %w", err)
- }
-
- staleIDs := s.collectStaleHelperIDsInternal(time.Now(), idleTimeout)
+ staleIDs := s.collectStaleHelperIDs(time.Now(), idleTimeout)
removed := 0
for _, containerID := range staleIDs {
- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions()); containerRemoveErr != nil {
+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions())
+ }); containerRemoveErr != nil {
slog.WarnContext(ctx, "failed to remove idle helper container", "containerId", containerID, "error", containerRemoveErr.Error())
continue
}
@@ -393,12 +350,9 @@ func (s *VolumeService) ReapIdleHelpers(ctx context.Context, idleTimeout time.Du
return removed, nil
}
-// collectStaleHelperIDsInternal removes idle (and any nil) entries from the helper
-// map and returns the container IDs that should be removed. Pure map/mutex
-// bookkeeping with no Docker calls, so it can be unit-tested directly. Entries are
-// dropped before their containers are removed so a concurrent request gets a clean
-// cache miss and re-creates a fresh helper.
-func (s *VolumeService) collectStaleHelperIDsInternal(now time.Time, idleTimeout time.Duration) []string {
+// collectStaleHelperIDs drops idle and nil entries and returns their container IDs.
+// Entries go before the containers so a concurrent request re-creates a fresh helper.
+func (s *VolumeService) collectStaleHelperIDs(now time.Time, idleTimeout time.Duration) []string {
staleIDs := make([]string, 0)
s.helperMu.Lock()
defer s.helperMu.Unlock()
@@ -407,9 +361,7 @@ func (s *VolumeService) collectStaleHelperIDsInternal(now time.Time, idleTimeout
delete(s.helperByVolume, volumeName)
continue
}
- // A helper with active holds is serving a request right now (e.g. a
- // download streaming for longer than the idle timeout); it becomes
- // reapable once released, since release refreshes lastUsedAt.
+ // Held helpers are serving a request; release refreshes lastUsedAt.
if helper.inUse > 0 {
continue
}
@@ -421,34 +373,28 @@ func (s *VolumeService) collectStaleHelperIDsInternal(now time.Time, idleTimeout
return staleIDs
}
-// StopHelper removes the reused helper for a single volume, if
-// one exists. It is idempotent: stopping a volume with no active helper returns
-// nil.
+// StopHelper removes the reused helper for a single volume; it is a no-op when none exists.
func (s *VolumeService) StopHelper(ctx context.Context, volumeName string) error {
if strings.TrimSpace(volumeName) == "" {
return nil
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return fmt.Errorf("failed to get docker client for helper stop: %w", err)
- }
-
- containerID := s.takeHelperIDInternal(volumeName)
+ containerID := s.takeHelperID(volumeName)
if containerID == "" {
return nil
}
- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions()); containerRemoveErr != nil {
+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, containerID, volumehelper.RemoveOptions())
+ }); containerRemoveErr != nil {
return fmt.Errorf("failed to remove helper container: %w", containerRemoveErr)
}
return nil
}
-// takeHelperIDInternal removes the helper entry for volumeName and returns its
-// container ID, or "" if there was none. Pure map/mutex bookkeeping.
-func (s *VolumeService) takeHelperIDInternal(volumeName string) string {
+// takeHelperID removes volumeName's helper entry and returns its container ID, or "".
+func (s *VolumeService) takeHelperID(volumeName string) string {
s.helperMu.Lock()
defer s.helperMu.Unlock()
helper := s.helperByVolume[volumeName]
@@ -462,23 +408,22 @@ func (s *VolumeService) takeHelperIDInternal(volumeName string) string {
func (s *VolumeService) CleanupOrphanedVolumeHelpers(ctx context.Context) (int, error) {
slog.DebugContext(ctx, "volume service: cleanup orphaned volume helper containers")
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return 0, fmt.Errorf("failed to get docker client for orphan helper cleanup: %w", err)
- }
-
- containers, err := dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ containers, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerListResult, error) {
+ return dockerClient.ContainerList(ctx, client.ContainerListOptions{All: true})
+ })
if err != nil {
return 0, fmt.Errorf("failed to list containers for orphan helper cleanup: %w", err)
}
removedCount := 0
for _, c := range containers.Items {
- if !isVolumeHelperContainerInternal(c) {
+ if !isVolumeHelperContainer(c) {
continue
}
- if _, containerRemoveErr := dockerClient.ContainerRemove(ctx, c.ID, volumehelper.RemoveOptions()); containerRemoveErr != nil {
+ if _, containerRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.ContainerRemoveResult, error) {
+ return dockerClient.ContainerRemove(ctx, c.ID, volumehelper.RemoveOptions())
+ }); containerRemoveErr != nil {
slog.WarnContext(
ctx,
"volume service: failed to remove orphaned volume helper container",
@@ -495,16 +440,7 @@ func (s *VolumeService) CleanupOrphanedVolumeHelpers(ctx context.Context) (int,
return removedCount, nil
}
-func (s *VolumeService) removeHelperEntry(volumeName string) {
- if strings.TrimSpace(volumeName) == "" {
- return
- }
- s.helperMu.Lock()
- delete(s.helperByVolume, volumeName)
- s.helperMu.Unlock()
-}
-
-func (s *VolumeService) execInContainerInternal(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error) {
+func (s *VolumeService) execInContainer(ctx context.Context, containerID, execUser string, cmd []string) (string, string, error) {
slog.DebugContext(ctx, "volume service: exec in container", "containerId", containerID, "user", execUser, "cmd", cmd)
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
diff --git a/backend/internal/volume/helpers_test.go b/backend/internal/volume/helpers_test.go
index 10268b803..28e48e08d 100644
--- a/backend/internal/volume/helpers_test.go
+++ b/backend/internal/volume/helpers_test.go
@@ -17,7 +17,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
)
-func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
+func newVolumeServiceTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
dockerClient, err := client.New(
client.WithHost(server.URL),
@@ -29,7 +29,7 @@ func newVolumeServiceTestDockerClientInternal(t *testing.T, server *httptest.Ser
return dockerClient
}
-func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {
+func TestCreateTempContainerReusesWritableHelper(t *testing.T) {
var createCalls, startCalls, removeCalls int
var createRequest struct {
NetworkDisabled bool `json:"NetworkDisabled"`
@@ -73,13 +73,13 @@ func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {
t.Cleanup(server.Close)
service := &VolumeService{
- dockerService: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeServiceTestDockerClientInternal(t, server)),
+ dockerService: docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeServiceTestDockerClient(t, server)),
helperByVolume: make(map[string]*volumeHelper),
}
- firstID, releaseFirst, err := service.acquireVolumeHelperInternal(t.Context(), "workspace-volume")
+ firstID, releaseFirst, err := service.acquireVolumeHelper(t.Context(), "workspace-volume")
require.NoError(t, err)
- secondID, releaseSecond, err := service.acquireVolumeHelperInternal(t.Context(), "workspace-volume")
+ secondID, releaseSecond, err := service.acquireVolumeHelper(t.Context(), "workspace-volume")
require.NoError(t, err)
require.Equal(t, "helper-1", firstID)
@@ -93,7 +93,7 @@ func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {
require.Equal(t, 2, service.helperByVolume["workspace-volume"].inUse)
service.helperByVolume["workspace-volume"].lastUsedAt = time.Now().Add(-time.Hour)
- require.Empty(t, service.collectStaleHelperIDsInternal(time.Now(), time.Minute))
+ require.Empty(t, service.collectStaleHelperIDs(time.Now(), time.Minute))
require.Contains(t, service.helperByVolume, "workspace-volume")
releaseFirst()
@@ -101,7 +101,7 @@ func TestCreateTempContainerInternalReusesWritableHelper(t *testing.T) {
require.Zero(t, service.helperByVolume["workspace-volume"].inUse)
}
-func TestIsUnlabeledVolumeHelperContainerInternal(t *testing.T) {
+func TestIsUnlabeledVolumeHelperContainer(t *testing.T) {
tests := []struct {
name string
summary container.Summary
@@ -137,12 +137,12 @@ func TestIsUnlabeledVolumeHelperContainerInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- require.Equal(t, tt.want, isUnlabeledVolumeHelperContainerInternal(tt.summary))
+ require.Equal(t, tt.want, isUnlabeledVolumeHelperContainer(tt.summary))
})
}
}
-func TestIsVolumeHelperContainerInternal_UsesExplicitHelperLabel(t *testing.T) {
+func TestIsVolumeHelperContainer_UsesExplicitHelperLabel(t *testing.T) {
tests := []struct {
name string
summary container.Summary
@@ -187,12 +187,12 @@ func TestIsVolumeHelperContainerInternal_UsesExplicitHelperLabel(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- require.Equal(t, tt.want, isVolumeHelperContainerInternal(tt.summary))
+ require.Equal(t, tt.want, isVolumeHelperContainer(tt.summary))
})
}
}
-func TestCollectStaleHelperIDsInternal(t *testing.T) {
+func TestCollectStaleHelperIDs(t *testing.T) {
now := time.Now()
s := &VolumeService{
helperByVolume: map[string]*volumeHelper{
@@ -203,7 +203,7 @@ func TestCollectStaleHelperIDsInternal(t *testing.T) {
},
}
- stale := s.collectStaleHelperIDsInternal(now, 10*time.Minute)
+ stale := s.collectStaleHelperIDs(now, 10*time.Minute)
require.ElementsMatch(t, []string{"c-stale", "c-atedge"}, stale,
"helpers idle >= timeout (and exactly at the edge) should be collected")
@@ -216,7 +216,7 @@ func TestCollectStaleHelperIDsInternal(t *testing.T) {
// A helper serving a request longer than the idle timeout (e.g. a slow download)
// must not be reaped mid-stream; it becomes collectible once released, with the
// idle clock measured from the release.
-func TestCollectStaleHelperIDsInternalSkipsInUseHelpers(t *testing.T) {
+func TestCollectStaleHelperIDsSkipsInUseHelpers(t *testing.T) {
now := time.Now()
s := &VolumeService{
helperByVolume: map[string]*volumeHelper{
@@ -224,26 +224,26 @@ func TestCollectStaleHelperIDsInternalSkipsInUseHelpers(t *testing.T) {
},
}
- release, ok := s.acquireHelperInternal("vol-a", "c-a")
+ release, ok := s.acquireHelper("vol-a", "c-a")
require.True(t, ok)
- require.Empty(t, s.collectStaleHelperIDsInternal(now, 10*time.Minute),
+ require.Empty(t, s.collectStaleHelperIDs(now, 10*time.Minute),
"an in-use helper must survive the reaper regardless of lastUsedAt")
require.Contains(t, s.helperByVolume, "vol-a")
release()
- require.Empty(t, s.collectStaleHelperIDsInternal(now, 10*time.Minute),
+ require.Empty(t, s.collectStaleHelperIDs(now, 10*time.Minute),
"release refreshes the idle clock, so the helper is fresh again")
- stale := s.collectStaleHelperIDsInternal(now.Add(11*time.Minute), 10*time.Minute)
+ stale := s.collectStaleHelperIDs(now.Add(11*time.Minute), 10*time.Minute)
require.Equal(t, []string{"c-a"}, stale)
// Acquiring a helper that was reaped (or replaced) since resolve must fail
// so the caller re-resolves instead of using a dead container.
- _, ok = s.acquireHelperInternal("vol-a", "c-a")
+ _, ok = s.acquireHelper("vol-a", "c-a")
require.False(t, ok)
}
-func TestTakeHelperIDInternal(t *testing.T) {
+func TestTakeHelperID(t *testing.T) {
s := &VolumeService{
helperByVolume: map[string]*volumeHelper{
"vol-a": {id: "c-a", lastUsedAt: time.Now()},
@@ -251,15 +251,15 @@ func TestTakeHelperIDInternal(t *testing.T) {
}
// Present: returns id and removes the entry.
- require.Equal(t, "c-a", s.takeHelperIDInternal("vol-a"))
+ require.Equal(t, "c-a", s.takeHelperID("vol-a"))
require.NotContains(t, s.helperByVolume, "vol-a")
// Absent (idempotent): returns "" without panicking.
- require.Empty(t, s.takeHelperIDInternal("vol-a"))
- require.Empty(t, s.takeHelperIDInternal("never-existed"))
+ require.Empty(t, s.takeHelperID("vol-a"))
+ require.Empty(t, s.takeHelperID("never-existed"))
}
-func TestTouchHelperInternal(t *testing.T) {
+func TestTouchHelper(t *testing.T) {
old := time.Now().Add(-30 * time.Minute)
s := &VolumeService{
helperByVolume: map[string]*volumeHelper{
@@ -267,11 +267,11 @@ func TestTouchHelperInternal(t *testing.T) {
},
}
- s.touchHelperInternal("vol-a")
+ s.touchHelper("vol-a")
require.True(t, s.helperByVolume["vol-a"].lastUsedAt.After(old),
"touch should reset the idle clock forward")
// Missing volume is a no-op (must not panic or create an entry).
- s.touchHelperInternal("missing")
+ s.touchHelper("missing")
require.NotContains(t, s.helperByVolume, "missing")
}
diff --git a/backend/internal/volume/service.go b/backend/internal/volume/service.go
index 8b9b8cd3e..1bd364666 100644
--- a/backend/internal/volume/service.go
+++ b/backend/internal/volume/service.go
@@ -8,6 +8,7 @@ import (
"log/slog"
"net/url"
"regexp"
+ "slices"
"strings"
"sync"
"time"
@@ -41,7 +42,6 @@ import (
volumebackup "github.com/getarcaneapp/arcane/backend/v2/internal/volume/children/backup"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume/children/workspace"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumes"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
@@ -112,7 +112,7 @@ func NewVolumeService(
helperByVolume: make(map[string]*volumeHelper),
}
backupDeps := volumebackup.Dependencies{
- Store: service.backupStoreInternal(),
+ Store: service.backupStore(),
DB: db,
Docker: dockerService,
Events: eventService,
@@ -122,9 +122,9 @@ func NewVolumeService(
RecoveryKeys: recoveryKeys,
Locks: &service.workspaceLocks,
AlreadyRunning: ErrVolumeBackupAlreadyRunning,
- HelperImage: service.getVolumeHelperImageInternal,
- AcquireHelper: service.acquireVolumeHelperInternal,
- Exec: service.execInContainerInternal,
+ HelperImage: service.getVolumeHelperImage,
+ AcquireHelper: service.acquireVolumeHelper,
+ Exec: service.execInContainer,
BackupVolumeName: backupVolumeName,
EncryptionKey: encryptionKey,
}
@@ -136,10 +136,10 @@ func NewVolumeService(
Docker: dockerService,
Events: eventService,
Locks: &service.workspaceLocks,
- AcquireHelper: service.acquireVolumeHelperInternal,
- RequireACFS: service.requireVolumeHelperACFSInternal,
- Exec: service.execInContainerInternal,
- HelperImage: service.getVolumeHelperImageInternal,
+ AcquireHelper: service.acquireVolumeHelper,
+ RequireACFS: service.requireVolumeHelperACFS,
+ Exec: service.execInContainer,
+ HelperImage: service.getVolumeHelperImage,
BackupStorageMount: service.backup.StorageMount,
ArchiveFileTarget: service.backup.ArchiveFileTarget,
RestoreArchiveMembers: service.backup.RestoreArchiveMembers,
@@ -171,19 +171,14 @@ func (s *VolumeService) HasEnabledBackupPolicy(ctx context.Context, volumeName s
}
// CreateSystemManagedBackup runs the existing volume backup workflow with a transient centralized policy.
-func (
- s *VolumeService,
-) CreateSystemManagedBackup(
+func (s *VolumeService) CreateSystemManagedBackup(
ctx context.Context,
volumeName string,
user usertypes.Actor,
trigger VolumeBackupTrigger,
policyID string,
policy backuptypes.UpdateBackupPolicy,
-) (
- *VolumeBackup,
- error,
-) {
+) (*VolumeBackup, error) {
entry, err := s.backup.CreateSystemManagedBackup(ctx, volumeName, user, string(trigger), policyID, policy)
if entry == nil {
return nil, err
@@ -224,7 +219,7 @@ func (s *VolumeService) MigrateRepositoryPasswords(ctx context.Context) error {
return s.backup.MigrateRepositoryPasswords(ctx)
}
-func (s *VolumeService) backupStoreInternal() volumebackup.Store {
+func (s *VolumeService) backupStore() volumebackup.Store {
return volumebackup.Store{
Create: func(ctx context.Context, record *volumetypes.Backup) error {
entry := volumeBackupFromRecordInternal(record)
@@ -277,7 +272,7 @@ func (s *VolumeService) backupStoreInternal() volumebackup.Store {
}
return entry.recordInternal(), nil
},
- List: s.listBackupsInternal,
+ List: s.listBackups,
RemoteSnapshotIDs: func(ctx context.Context, destinationID string) ([]string, error) {
var knownIDs []string
err := s.db.WithContext(ctx).Model(&VolumeBackup{}).
@@ -288,8 +283,8 @@ func (s *VolumeService) backupStoreInternal() volumebackup.Store {
}
}
-// listBackupsInternal pages one volume's backup history; both or neither management type selected means no type filter.
-func (s *VolumeService) listBackupsInternal(ctx context.Context, volumeName string, params pagination.QueryParams) ([]volumetypes.Backup, int64, error) {
+// listBackups pages one volume's backup history; both or neither management type selected means no type filter.
+func (s *VolumeService) listBackups(ctx context.Context, volumeName string, params pagination.QueryParams) ([]volumetypes.Backup, int64, error) {
query := s.db.WithContext(ctx).Model(&VolumeBackup{}).Where("volume_name = ?", volumeName)
var system, managed bool
for value := range strings.SplitSeq(params.Filters["type"], ",") {
@@ -373,33 +368,20 @@ func (s *VolumeService) listBackupsInternal(ctx context.Context, volumeName stri
func (s *VolumeService) GetVolumeByName(ctx context.Context, name string) (*volumetypes.Volume, error) {
slog.DebugContext(ctx, "volume service: get volume", "volume", name)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- volResult, err := dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})
+ volResult, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("volume not found: %w", err)
}
- vol := volResult.Volume
-
- localSettings := s.settingsService.GetSettingsConfig()
- usageCtx, usageCancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
- defer usageCancel()
- if usageVolumes, ok := docker.GetVolumeUsageDataStaleWhileRevalidate(usageCtx, dockerClient).Get(); ok {
- for _, uv := range usageVolumes {
- if uv.Name == vol.Name && uv.UsageData != nil {
- vol.UsageData = uv.UsageData
- slog.DebugContext(ctx, "attached volume usage data", "volume", vol.Name, "sizeBytes", uv.UsageData.Size, "refCount", uv.UsageData.RefCount)
- break
- }
- }
- }
-
- v := volumetypes.NewSummary(vol)
+ usage, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (mo.Option[[]volume.Volume], error) {
+ return docker.GetVolumeUsageDataStaleWhileRevalidate(ctx, dockerClient), nil
+ })
+ v := volumetypes.NewSummary(enrichVolumesWithUsageData([]volume.Volume{volResult.Volume}, usage.OrEmpty())[0])
- containerIDs, err := docker.GetContainersUsingVolume(ctx, dockerClient, name)
+ containerIDs, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]string, error) {
+ return docker.GetContainersUsingVolume(ctx, dockerClient, name)
+ })
if err != nil {
slog.WarnContext(ctx, "failed to get containers using volume", "volume", name, "error", err.Error())
} else {
@@ -421,14 +403,18 @@ func (s *VolumeService) CreateVolume(ctx context.Context, options client.VolumeC
}
defer s.eventService.BeginDockerResourceSuppressionWindow("volume", options.Name, options.Name)()
- created, err := dockerClient.VolumeCreate(ctx, options)
+ created, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeCreateResult, error) {
+ return dockerClient.VolumeCreate(ctx, options)
+ })
if err != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", "", options.Name, user.ID, user.Username, "0", err, database.JSON{"action": "create", "driver": options.Driver})
return nil, fmt.Errorf("failed to create volume: %w", err)
}
defer s.eventService.BeginDockerResourceSuppressionWindow("volume", created.Volume.Name, created.Volume.Name)()
- vol, err := dockerClient.VolumeInspect(ctx, created.Volume.Name, client.VolumeInspectOptions{})
+ vol, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, created.Volume.Name, client.VolumeInspectOptions{})
+ })
if err != nil {
s.eventService.LogErrorEvent(
ctx,
@@ -482,8 +468,8 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boo
}
defer s.eventService.BeginDockerResourceSuppressionWindow("volume", name, name)()
- if _, volumeRemoveErr := dockerClient.VolumeRemove(ctx, name, client.VolumeRemoveOptions{
- Force: force,
+ if _, volumeRemoveErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeRemoveResult, error) {
+ return dockerClient.VolumeRemove(ctx, name, client.VolumeRemoveOptions{Force: force})
}); volumeRemoveErr != nil {
s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", name, name, user.ID, user.Username, "0", volumeRemoveErr, database.JSON{"action": "delete", "force": force})
return fmt.Errorf("failed to remove volume: %w", volumeRemoveErr)
@@ -497,7 +483,7 @@ func (s *VolumeService) DeleteVolume(ctx context.Context, name string, force boo
slog.WarnContext(ctx, "could not log volume deletion action", "volume", name, "error", logErr.Error())
}
- s.removeHelperEntry(name)
+ s.takeHelperID(name)
s.backup.RemovePolicies(ctx, name)
docker.InvalidateVolumeUsageCache(dockerClient)
return nil
@@ -524,18 +510,27 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (
// - Without 'all' flag: Only removes anonymous (unnamed) volumes that are not in use
// - With 'all=true' flag: Removes ALL unused volumes (both named and anonymous)
// Note: Volumes are considered "in use" if referenced by any container (running or stopped)
- volumePruneResult, err := dockerClient.VolumePrune(ctx, buildVolumePruneOptionsInternal(all))
+ volumePruneResult, err := dockerClient.VolumePrune(ctx, client.VolumePruneOptions{
+ All: all,
+ Filters: make(client.Filters).Add("label!", internalVolumePruneFilterValue),
+ })
if err != nil {
return nil, fmt.Errorf("failed to prune volumes: %w", err)
}
- metadata := buildVolumePruneMetadataInternal(all, len(volumePruneResult.Report.VolumesDeleted), volumePruneResult.Report.SpaceReclaimed)
+ metadata := database.JSON{
+ "action": "prune",
+ "all": all,
+ "volumesDeleted": len(volumePruneResult.Report.VolumesDeleted),
+ "spaceReclaimed": volumePruneResult.Report.SpaceReclaimed,
+ "internalVolumeFilterLabel": internalVolumePruneFilterValue,
+ }
if logErr := s.eventService.LogVolumeEvent(ctx, event.EventTypeVolumeDelete, "", "bulk_prune", usertypes.SystemUser.ID, usertypes.SystemUser.Username, "0", metadata); logErr != nil {
slog.WarnContext(ctx, "could not log volume prune action", "error", logErr.Error())
}
for _, volumeName := range volumePruneResult.Report.VolumesDeleted {
- s.removeHelperEntry(volumeName)
+ s.takeHelperID(volumeName)
s.backup.RemovePolicies(ctx, volumeName)
}
@@ -547,35 +542,18 @@ func (s *VolumeService) PruneVolumesWithOptions(ctx context.Context, all bool) (
}, nil
}
-func buildVolumePruneOptionsInternal(all bool) client.VolumePruneOptions {
- filters := make(client.Filters)
- filters = filters.Add("label!", internalVolumePruneFilterValue)
- return client.VolumePruneOptions{All: all, Filters: filters}
-}
-
-func buildVolumePruneMetadataInternal(all bool, volumesDeleted int, spaceReclaimed uint64) database.JSON {
- return database.JSON{
- "action": "prune",
- "all": all,
- "volumesDeleted": volumesDeleted,
- "spaceReclaimed": spaceReclaimed,
- "internalVolumeFilterLabel": internalVolumePruneFilterValue,
- }
-}
-
func (s *VolumeService) GetVolumeUsage(ctx context.Context, name string) (bool, []string, error) {
slog.DebugContext(ctx, "volume service: get volume usage", "volume", name)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return false, nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- vol, err := dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})
+ vol, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, name, client.VolumeInspectOptions{})
+ })
if err != nil {
return false, nil, fmt.Errorf("volume not found: %w", err)
}
- containerIDs, err := docker.GetContainersUsingVolume(ctx, dockerClient, vol.Volume.Name)
+ containerIDs, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) ([]string, error) {
+ return docker.GetContainersUsingVolume(ctx, dockerClient, vol.Volume.Name)
+ })
if err != nil {
return false, nil, fmt.Errorf("failed to get containers using volume: %w", err)
}
@@ -594,16 +572,7 @@ type VolumeSizeData struct {
// This is a slow operation as it calls Docker's DiskUsage API.
func (s *VolumeService) GetVolumeSizes(ctx context.Context) (map[string]VolumeSizeData, error) {
slog.DebugContext(ctx, "volume service: get volume sizes")
- localSettings := s.settingsService.GetSettingsConfig()
- apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
- defer cancel()
-
- dockerClient, err := s.dockerService.GetClient(apiCtx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
-
- usageVolumes, err := docker.GetVolumeUsageData(apiCtx, dockerClient)
+ usageVolumes, err := s.dockerService.Call(ctx, docker.GetVolumeUsageData)
if err != nil {
return nil, fmt.Errorf("failed to get volume usage data: %w", err)
}
@@ -621,7 +590,7 @@ func (s *VolumeService) GetVolumeSizes(ctx context.Context) (map[string]VolumeSi
return result, nil
}
-func enrichVolumesWithUsageDataInternal(dockerVolumes, usageVolumes []volume.Volume) []volume.Volume {
+func enrichVolumesWithUsageData(dockerVolumes, usageVolumes []volume.Volume) []volume.Volume {
usageByName := make(map[string]*volume.UsageData, len(usageVolumes))
for _, uv := range usageVolumes {
if uv.Name == "" || uv.UsageData == nil {
@@ -644,25 +613,24 @@ func enrichVolumesWithUsageDataInternal(dockerVolumes, usageVolumes []volume.Vol
return result
}
-func (s *VolumeService) buildVolumeContainerMapInternal(ctx context.Context) (map[string][]string, error) {
- containers, err := s.dockerService.ListContainers(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to list containers: %w", err)
- }
-
- volumeContainerMap := make(map[string][]string)
- for _, c := range containers {
- for _, m := range c.Mounts {
- if m.Type == mount.TypeVolume && m.Name != "" {
- volumeContainerMap[m.Name] = append(volumeContainerMap[m.Name], c.ID)
+func (s *VolumeService) buildVolumePaginationConfig() pagination.Config[volumetypes.Volume] {
+ createdSortFn := func(a, b volumetypes.Volume) int {
+ parseCreatedAt := func(createdAt string) (time.Time, bool) {
+ for _, layout := range []string{time.RFC3339Nano, time.RFC3339} {
+ if parsed, err := time.Parse(layout, createdAt); err == nil {
+ return parsed, true
+ }
}
+ return time.Time{}, false
+ }
+ aTime, aOk := parseCreatedAt(a.CreatedAt)
+ bTime, bOk := parseCreatedAt(b.CreatedAt)
+ if aOk && bOk {
+ return aTime.Compare(bTime)
}
+ return strings.Compare(a.CreatedAt, b.CreatedAt)
}
- return volumeContainerMap, nil
-}
-
-func (s *VolumeService) buildVolumePaginationConfig() pagination.Config[volumetypes.Volume] {
return pagination.Config[volumetypes.Volume]{
SearchAccessors: []pagination.SearchAccessor[volumetypes.Volume]{
func(v volumetypes.Volume) (string, error) { return v.Name, nil },
@@ -670,127 +638,62 @@ func (s *VolumeService) buildVolumePaginationConfig() pagination.Config[volumety
func(v volumetypes.Volume) (string, error) { return v.Mountpoint, nil },
func(v volumetypes.Volume) (string, error) { return v.Scope, nil },
},
- SortBindings: s.buildVolumeSortBindingsInternal(),
- FilterAccessors: buildVolumeFilterAccessorsInternal(),
- }
-}
-
-func (s *VolumeService) buildVolumeSortBindingsInternal() []pagination.SortBinding[volumetypes.Volume] {
- createdSortFn := s.compareVolumeCreatedInternal
-
- return []pagination.SortBinding[volumetypes.Volume]{
- {
- Key: "name",
- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Name, b.Name) },
- },
- {
- Key: "driver",
- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Driver, b.Driver) },
- },
- {
- Key: "mountpoint",
- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Mountpoint, b.Mountpoint) },
- },
- {
- Key: "scope",
- Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Scope, b.Scope) },
- },
- {
- Key: "created",
- Fn: createdSortFn,
- },
- {
- Key: "createdAt",
- Fn: createdSortFn,
- },
- {
- Key: "inUse",
- Fn: func(a, b volumetypes.Volume) int {
- if a.InUse == b.InUse {
- return 0
- }
- return kit.Ternary(a.InUse, -1, 1)
+ SortBindings: []pagination.SortBinding[volumetypes.Volume]{
+ {
+ Key: "name",
+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Name, b.Name) },
+ },
+ {
+ Key: "driver",
+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Driver, b.Driver) },
+ },
+ {
+ Key: "mountpoint",
+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Mountpoint, b.Mountpoint) },
+ },
+ {
+ Key: "scope",
+ Fn: func(a, b volumetypes.Volume) int { return strings.Compare(a.Scope, b.Scope) },
+ },
+ {
+ Key: "created",
+ Fn: createdSortFn,
+ },
+ {
+ Key: "createdAt",
+ Fn: createdSortFn,
+ },
+ {
+ Key: "inUse",
+ Fn: func(a, b volumetypes.Volume) int {
+ if a.InUse == b.InUse {
+ return 0
+ }
+ return kit.Ternary(a.InUse, -1, 1)
+ },
+ },
+ {
+ Key: "size",
+ Fn: func(a, b volumetypes.Volume) int {
+ aSize := cmp.Or(a.Size, kit.FromPtr(a.UsageData).Size)
+ bSize := cmp.Or(b.Size, kit.FromPtr(b.UsageData).Size)
+ return cmp.Or(cmp.Compare(aSize, bSize), strings.Compare(a.Name, b.Name))
+ },
},
},
- {
- Key: "size",
- Fn: compareVolumeSizesInternal,
- },
- }
-}
-
-func compareVolumeSizesInternal(a, b volumetypes.Volume) int {
- aSize := a.Size
- bSize := b.Size
-
- if aSize == 0 && a.UsageData != nil {
- aSize = a.UsageData.Size
- }
- if bSize == 0 && b.UsageData != nil {
- bSize = b.UsageData.Size
- }
-
- if aSize == bSize {
- return strings.Compare(a.Name, b.Name)
- }
- return kit.Ternary(aSize < bSize, -1, 1)
-}
-
-func (s *VolumeService) compareVolumeCreatedInternal(a, b volumetypes.Volume) int {
- aTime, aOk := parseVolumeCreatedAtInternal(a.CreatedAt).Get()
- bTime, bOk := parseVolumeCreatedAtInternal(b.CreatedAt).Get()
- if aOk && bOk {
- if aTime.Before(bTime) {
- return -1
- }
- return kit.Ternary(aTime.After(bTime), 1, 0)
- }
- return strings.Compare(a.CreatedAt, b.CreatedAt)
-}
-
-func parseVolumeCreatedAtInternal(createdAt string) mo.Option[time.Time] {
- if createdAt == "" {
- return mo.None[time.Time]()
- }
- if parsed, err := time.Parse(time.RFC3339Nano, createdAt); err == nil {
- return mo.Some(parsed)
- }
- if parsed, err := time.Parse(time.RFC3339, createdAt); err == nil {
- return mo.Some(parsed)
- }
- return mo.None[time.Time]()
-}
-
-func buildVolumeFilterAccessorsInternal() []pagination.FilterAccessor[volumetypes.Volume] {
- return []pagination.FilterAccessor[volumetypes.Volume]{
- {
- Key: "inUse",
- Fn: func(v volumetypes.Volume, filterValue string) bool {
- return kit.Ternary(filterValue == "true", v.InUse, filterValue != "false" || !v.InUse)
+ FilterAccessors: []pagination.FilterAccessor[volumetypes.Volume]{
+ {
+ Key: "inUse",
+ Fn: func(v volumetypes.Volume, filterValue string) bool {
+ return kit.Ternary(filterValue == "true", v.InUse, filterValue != "false" || !v.InUse)
+ },
},
},
}
}
-func calculateVolumeUsageCountsInternal(items []volumetypes.Volume) volumetypes.UsageCounts {
- counts := volumetypes.UsageCounts{
- Total: len(items),
- }
- for _, v := range items {
- if v.InUse {
- counts.Inuse++
- } else {
- counts.Unused++
- }
- }
- return counts
-}
-
-// CountUsageFromSnapshot derives volume usage counts from an
-// already-fetched volume and container listing. It mirrors the semantics of
-// ListVolumesPaginated — internal volumes are excluded and a volume counts as
-// in use when any container mounts it — so dashboard tile numbers agree with
-// the volumes page.
+// CountUsageFromSnapshot derives usage counts from fetched volumes and containers with ListVolumesPaginated's
+// semantics (internal volumes excluded, mounted means in use) so dashboard tiles match the volumes page.
func (s *VolumeService) CountUsageFromSnapshot(dockerVolumes []volume.Volume, containers []container.Summary) volumetypes.UsageCounts {
inUse := make(map[string]struct{})
for _, c := range containers {
@@ -803,7 +706,7 @@ func (s *VolumeService) CountUsageFromSnapshot(dockerVolumes []volume.Volume, co
counts := volumetypes.UsageCounts{}
for _, v := range dockerVolumes {
- if s.isInternalVolumeInternal(volumetypes.NewSummary(v)) {
+ if s.isInternalVolume(volumetypes.NewSummary(v)) {
continue
}
counts.Total++
@@ -821,7 +724,7 @@ func (s *VolumeService) IsBackupVolume(name string) bool {
return strings.EqualFold(strings.TrimSpace(name), strings.TrimSpace(s.backupVolumeName))
}
-func (s *VolumeService) isInternalVolumeInternal(v volumetypes.Volume) bool {
+func (s *VolumeService) isInternalVolume(v volumetypes.Volume) bool {
if s.IsBackupVolume(v.Name) {
return true
}
@@ -830,55 +733,33 @@ func (s *VolumeService) isInternalVolumeInternal(v volumetypes.Volume) bool {
return internal
}
-var generatedAnonymousVolumeNameInternal = regexp.MustCompile(`^[a-f0-9]{64}$`)
-
-func isAnonymousVolumeInternal(v volume.Volume) bool {
- if _, ok := v.Labels["com.docker.volume.anonymous"]; ok {
- return true
- }
- return generatedAnonymousVolumeNameInternal.MatchString(v.Name)
-}
-
-func mountedVolumeNamesInternal(mounts []container.MountPoint) map[string]struct{} {
- names := make(map[string]struct{})
- for _, item := range mounts {
- if item.Type != mount.TypeVolume {
- continue
- }
- name := cmp.Or(strings.TrimSpace(item.Name), strings.TrimSpace(item.Source))
- if name != "" {
- names[name] = struct{}{}
- }
- }
- return names
-}
+var generatedAnonymousVolumeName = regexp.MustCompile(`^[a-f0-9]{64}$`)
// ListBackupVolumeOptions returns the current non-internal volumes used by centralized backup selection.
func (s *VolumeService) ListBackupVolumeOptions(ctx context.Context) ([]backuptypes.SystemVolumeBackupOption, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
- result, err := dockerClient.VolumeList(ctx, client.VolumeListOptions{})
+ result, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeListResult, error) {
+ return dockerClient.VolumeList(ctx, client.VolumeListOptions{})
+ })
if err != nil {
return nil, fmt.Errorf("failed to list Docker volumes: %w", err)
}
- arcaneVolumes := make(map[string]struct{})
- if inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr == nil && inspect != nil {
- arcaneVolumes = mountedVolumeNamesInternal(inspect.Mounts)
- } else if inspectErr != nil {
+ inspect, inspectErr := s.dockerService.Call(ctx, libarcane.InspectCurrentArcaneContainer)
+ if inspectErr != nil {
slog.DebugContext(ctx, "volume service: could not identify Arcane-mounted volumes for centralized backups", "error", inspectErr)
}
+ arcaneMounts := kit.FromPtr(inspect).Mounts
options := make([]backuptypes.SystemVolumeBackupOption, 0, len(result.Items))
for _, item := range result.Items {
- if s.isInternalVolumeInternal(volumetypes.NewSummary(item)) {
- continue
- }
- if _, isArcaneVolume := arcaneVolumes[item.Name]; isArcaneVolume {
+ if s.isInternalVolume(volumetypes.NewSummary(item)) || slices.ContainsFunc(arcaneMounts, func(m container.MountPoint) bool {
+ return m.Type == mount.TypeVolume && cmp.Or(strings.TrimSpace(m.Name), strings.TrimSpace(m.Source)) == item.Name
+ }) {
continue
}
+ _, anonymousLabel := item.Labels["com.docker.volume.anonymous"]
options = append(options, backuptypes.SystemVolumeBackupOption{
- Name: item.Name, Anonymous: isAnonymousVolumeInternal(item), Available: true,
+ Name: item.Name,
+ Anonymous: anonymousLabel || generatedAnonymousVolumeName.MatchString(item.Name),
+ Available: true,
})
}
return options, nil
@@ -902,10 +783,6 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat
"includeInternal",
includeInternal,
)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, pagination.Response{}, volumetypes.UsageCounts{}, fmt.Errorf("failed to connect to Docker: %w", err)
- }
// Run volume list and container list in parallel for better performance
type volumeListResult struct {
@@ -920,19 +797,29 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat
volChan := make(chan volumeListResult, 1)
containerChan := make(chan containerMapResult, 1)
- localSettings := s.settingsService.GetSettingsConfig()
- apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(localSettings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))
- defer cancel()
-
- go func(ctx context.Context) {
- volListBody, volumeListErr := dockerClient.VolumeList(ctx, client.VolumeListOptions{})
+ go func() {
+ volListBody, volumeListErr := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeListResult, error) {
+ return dockerClient.VolumeList(ctx, client.VolumeListOptions{})
+ })
volChan <- volumeListResult{volumes: volListBody.Items, err: volumeListErr}
- }(apiCtx)
+ }()
- go func(ctx context.Context) {
- containerMap, buildVolumeContainerMapErr := s.buildVolumeContainerMapInternal(ctx)
- containerChan <- containerMapResult{containerMap: containerMap, err: buildVolumeContainerMapErr}
- }(apiCtx)
+ go func() {
+ containers, listErr := s.dockerService.ListContainers(ctx)
+ if listErr != nil {
+ containerChan <- containerMapResult{err: fmt.Errorf("failed to list containers: %w", listErr)}
+ return
+ }
+ containerMap := make(map[string][]string)
+ for _, c := range containers {
+ for _, m := range c.Mounts {
+ if m.Type == mount.TypeVolume && m.Name != "" {
+ containerMap[m.Name] = append(containerMap[m.Name], c.ID)
+ }
+ }
+ }
+ containerChan <- containerMapResult{containerMap: containerMap}
+ }()
// Wait for both results
volResult := <-volChan
@@ -954,7 +841,10 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat
// background so this list request never waits for Docker's DiskUsage call.
var usageVolumes []volume.Volume
if params.Sort == "size" {
- if uv, found := docker.GetVolumeUsageDataStaleWhileRevalidate(apiCtx, dockerClient).Get(); found && (len(uv) > 0 || len(volResult.volumes) == 0) {
+ usage, _ := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (mo.Option[[]volume.Volume], error) {
+ return docker.GetVolumeUsageDataStaleWhileRevalidate(ctx, dockerClient), nil
+ })
+ if uv, found := usage.Get(); found && (len(uv) > 0 || len(volResult.volumes) == 0) {
usageVolumes = uv
usageCacheSnapshot = "available"
} else {
@@ -964,12 +854,12 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat
}
}
- enrichedVolumes := enrichVolumesWithUsageDataInternal(volResult.volumes, usageVolumes)
+ enrichedVolumes := enrichVolumesWithUsageData(volResult.volumes, usageVolumes)
items := make([]volumetypes.Volume, 0, len(enrichedVolumes))
for _, v := range enrichedVolumes {
volDto := volumetypes.NewSummary(v)
- if !includeInternal && s.isInternalVolumeInternal(volDto) {
+ if !includeInternal && s.isInternalVolume(volDto) {
continue
}
if containerIDs, ok := volumeContainerMap[v.Name]; ok {
@@ -981,13 +871,20 @@ func (s *VolumeService) ListVolumesPaginated(ctx context.Context, params paginat
items = append(items, volDto)
}
- paginationConfig := s.buildVolumePaginationConfig()
- result := paginationConfig.SearchOrderAndPaginate(items, effectiveParams)
- counts := calculateVolumeUsageCountsInternal(items)
+ result := s.buildVolumePaginationConfig().SearchOrderAndPaginate(items, effectiveParams)
+ inUseCount := 0
+ for _, v := range items {
+ inUseCount += kit.Ternary(v.InUse, 1, 0)
+ }
+ counts := volumetypes.UsageCounts{
+ Inuse: inUseCount,
+ Unused: len(items) - inUseCount,
+ Total: len(items),
+ }
paginationResp := pagination.BuildResponse(result.TotalCount, result.TotalAvailable, effectiveParams)
slog.DebugContext(
ctx, "volume service: listed volumes",
- "dockerHost", dockerClient.DaemonHost(),
+ "dockerHost", s.dockerService.DockerHost(),
"requestedSort", params.Sort,
"requestedOrder", params.Order,
"effectiveSort", effectiveParams.Sort,
@@ -1070,19 +967,21 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strin
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "connect", err)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "connect", err)
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
- source, err := dockerClient.VolumeInspect(ctx, oldName, client.VolumeInspectOptions{})
+ source, err := s.dockerService.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (client.VolumeInspectResult, error) {
+ return dockerClient.VolumeInspect(ctx, oldName, client.VolumeInspectOptions{})
+ })
if err != nil {
if errdefs.IsNotFound(err) {
err = common.Classify(common.ErrNotFound, err)
}
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "inspect", err)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "inspect", err)
return nil, fmt.Errorf("inspect source volume: %w", err)
}
- if s.isInternalVolumeInternal(volumetypes.NewSummary(source.Volume)) {
+ if s.isInternalVolume(volumetypes.NewSummary(source.Volume)) {
return nil, common.ErrVolumeRenameProtected
}
@@ -1092,23 +991,23 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strin
slog.WarnContext(ctx, "could not stop volume browse helper before rename", "volume", oldName, "error", stopErr.Error())
}
- migration, err := volumes.PlanRename(ctx, dockerClient, oldName, newName, s.toolsImageInternal())
+ migration, err := volumes.PlanRename(ctx, dockerClient, oldName, newName, s.toolsImage())
if err != nil {
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "plan", err)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "plan", err)
return nil, err
}
if applyErr := migration.Apply(ctx); applyErr != nil {
if errdefs.IsInvalidArgument(applyErr) {
applyErr = common.Classify(common.ErrBadRequest, applyErr)
}
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "apply", applyErr)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "apply", applyErr)
return nil, applyErr
}
- if renameVolumeMetadataErr := s.renameVolumeMetadataInternal(ctx, oldName, newName); renameVolumeMetadataErr != nil {
+ if renameVolumeMetadataErr := s.renameVolumeMetadata(ctx, oldName, newName); renameVolumeMetadataErr != nil {
rollbackErr := migration.Rollback(ctx)
combinedErr := errors.Join(fmt.Errorf("rename volume metadata: %w", renameVolumeMetadataErr), rollbackErr)
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "metadata", combinedErr)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "metadata", combinedErr)
return nil, combinedErr
}
@@ -1119,21 +1018,21 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strin
// failed, so the rename itself succeeded.
slog.WarnContext(ctx, "volume renamed but source volume could not be removed", "oldVolume", oldName, "newVolume", newName, "error", commitErr.Error())
} else {
- metadataErr := s.renameVolumeMetadataInternal(ctx, newName, oldName)
+ metadataErr := s.renameVolumeMetadata(ctx, newName, oldName)
rollbackErr := migration.Rollback(ctx)
combinedErr := errors.Join(commitErr, metadataErr, rollbackErr)
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "commit", combinedErr)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "commit", combinedErr)
return nil, combinedErr
}
}
}
- s.removeHelperEntry(oldName)
+ s.takeHelperID(oldName)
docker.InvalidateVolumeUsageCache(dockerClient)
renamed, err := s.GetVolumeByName(ctx, newName)
if err != nil {
- s.logVolumeRenameErrorInternal(ctx, oldName, newName, user, "inspect-renamed", err)
+ s.logVolumeRenameError(ctx, oldName, newName, user, "inspect-renamed", err)
return nil, fmt.Errorf("inspect renamed volume: %w", err)
}
@@ -1147,7 +1046,7 @@ func (s *VolumeService) RenameVolume(ctx context.Context, oldName, newName strin
// Backup jobs are keyed by policy ID and re-read the policy row on every run,
// so renaming volume_name requires no job rescheduling.
-func (s *VolumeService) renameVolumeMetadataInternal(ctx context.Context, oldName, newName string) error {
+func (s *VolumeService) renameVolumeMetadata(ctx context.Context, oldName, newName string) error {
if s.db == nil {
return nil
}
@@ -1163,7 +1062,7 @@ func (s *VolumeService) renameVolumeMetadataInternal(ctx context.Context, oldNam
})
}
-func (s *VolumeService) logVolumeRenameErrorInternal(ctx context.Context, oldName, newName string, user usertypes.Actor, step string, err error) {
+func (s *VolumeService) logVolumeRenameError(ctx context.Context, oldName, newName string, user usertypes.Actor, step string, err error) {
s.eventService.LogErrorEvent(ctx, event.EventTypeVolumeError, "volume", oldName, oldName, user.ID, user.Username, "0", err, database.JSON{
"action": "rename",
"step": step,
diff --git a/backend/internal/volume/service_test.go b/backend/internal/volume/service_test.go
index f55e3543a..c3162c249 100644
--- a/backend/internal/volume/service_test.go
+++ b/backend/internal/volume/service_test.go
@@ -39,7 +39,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
)
-func TestEnrichVolumesWithUsageDataInternal(t *testing.T) {
+func TestEnrichVolumesWithUsageData(t *testing.T) {
tests := []struct {
name string
volumes []volume.Volume
@@ -85,18 +85,18 @@ func TestEnrichVolumesWithUsageDataInternal(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := enrichVolumesWithUsageDataInternal(tt.volumes, tt.usageVolumes)
+ got := enrichVolumesWithUsageData(tt.volumes, tt.usageVolumes)
require.Len(t, got, tt.wantLen)
tt.assertions(t, got)
})
}
}
-func TestIsInternalVolumeInternal(t *testing.T) {
+func TestIsInternalVolume(t *testing.T) {
svc := &VolumeService{backupVolumeName: "arcane-backups"}
- require.True(t, svc.isInternalVolumeInternal(volumetypes.Volume{Name: "arcane-backups"}))
- require.False(t, svc.isInternalVolumeInternal(volumetypes.Volume{Name: "user-volume"}))
+ require.True(t, svc.isInternalVolume(volumetypes.Volume{Name: "arcane-backups"}))
+ require.False(t, svc.isInternalVolume(volumetypes.Volume{Name: "user-volume"}))
}
func TestRenameVolumeRejectsInvalidAndProtectedNames(t *testing.T) {
@@ -112,7 +112,7 @@ func TestRenameVolumeRejectsInvalidAndProtectedNames(t *testing.T) {
require.ErrorIs(t, err, common.ErrVolumeRenameProtected)
}
-func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {
+func applyVolumeBackupMigrations(t *testing.T, gormDB *gorm.DB) {
t.Helper()
for _, name := range []string{"032_add_volume_backups.sql", "073_add_backup_support.sql", "087_add_volume_backup_remote_instance.sql"} {
migration, err := os.ReadFile("../../resources/migrations/sqlite/" + name)
@@ -121,15 +121,15 @@ func applyVolumeBackupMigrationsInternal(t *testing.T, gormDB *gorm.DB) {
}
}
-func TestRenameVolumeMetadataInternalPreservesPoliciesAndHistory(t *testing.T) {
+func TestRenameVolumeMetadataPreservesPoliciesAndHistory(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:volume-rename-metadata?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- applyVolumeBackupMigrationsInternal(t, gormDB)
+ applyVolumeBackupMigrations(t, gormDB)
require.NoError(t, gormDB.Exec("INSERT INTO volume_backup_policies (id, volume_name, schedule) VALUES (?, ?, ?)", "policy-1", "source-data", "0 0 2 * * *").Error)
require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "source-data", PolicyID: "policy-1"}).Error)
service := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil)
- require.NoError(t, service.renameVolumeMetadataInternal(t.Context(), "source-data", "renamed-data"))
+ require.NoError(t, service.renameVolumeMetadata(t.Context(), "source-data", "renamed-data"))
var renamedPolicy string
require.NoError(t, gormDB.Raw("SELECT volume_name FROM volume_backup_policies WHERE id = ?", "policy-1").Scan(&renamedPolicy).Error)
@@ -140,33 +140,7 @@ func TestRenameVolumeMetadataInternalPreservesPoliciesAndHistory(t *testing.T) {
require.Equal(t, "renamed-data", backupEntry.VolumeName)
}
-func TestBuildVolumePruneOptionsInternal_PreservesInternalVolumes(t *testing.T) {
- options := buildVolumePruneOptionsInternal(true)
-
- require.True(t, options.All)
- require.NotNil(t, options.Filters)
- require.True(t, options.Filters["label!"][internalVolumePruneFilterValue])
-}
-
-func TestBuildVolumePruneOptionsInternal_PreservesInternalVolumesForAnonymousVolumes(t *testing.T) {
- options := buildVolumePruneOptionsInternal(false)
-
- require.False(t, options.All)
- require.NotNil(t, options.Filters)
- require.True(t, options.Filters["label!"][internalVolumePruneFilterValue])
-}
-
-func TestBuildVolumePruneMetadataInternal(t *testing.T) {
- metadata := buildVolumePruneMetadataInternal(true, 2, 4096)
-
- require.Equal(t, "prune", metadata["action"])
- require.Equal(t, true, metadata["all"])
- require.Equal(t, 2, metadata["volumesDeleted"])
- require.EqualValues(t, 4096, metadata["spaceReclaimed"])
- require.Equal(t, internalVolumePruneFilterValue, metadata["internalVolumeFilterLabel"])
-}
-
-func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {
+func TestListBackupsPaginatedByManagementType(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-management?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, gormDB.AutoMigrate(&VolumeBackup{}))
@@ -179,7 +153,7 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {
}
service := &VolumeService{db: &database.DB{DB: gormDB}}
- systemRows, total, err := service.listBackupsInternal(t.Context(), "app", pagination.QueryParams{
+ systemRows, total, err := service.listBackups(t.Context(), "app", pagination.QueryParams{
Filters: map[string]string{"type": "system"},
})
require.NoError(t, err)
@@ -187,7 +161,7 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {
require.Len(t, systemRows, 1)
require.Equal(t, backup.ManagementTypeSystem, systemRows[0].Type)
- volumeRows, total, err := service.listBackupsInternal(t.Context(), "app", pagination.QueryParams{
+ volumeRows, total, err := service.listBackups(t.Context(), "app", pagination.QueryParams{
Filters: map[string]string{"type": "volume"},
})
require.NoError(t, err)
@@ -198,7 +172,7 @@ func TestListBackupsPaginatedByManagementTypeInternal(t *testing.T) {
}
}
-func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.Server) *client.Client {
+func newVolumeWorkspaceTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
t.Helper()
dialer := &net.Dialer{}
dockerClient, err := client.New(
@@ -214,7 +188,7 @@ func newVolumeWorkspaceTestDockerClientInternal(t *testing.T, server *httptest.S
return dockerClient
}
-func writeDockerExecAttachResponseInternal(t *testing.T, w http.ResponseWriter, stdout string) {
+func writeDockerExecAttachResponse(t *testing.T, w http.ResponseWriter, stdout string) {
t.Helper()
hijacker, ok := w.(http.Hijacker)
if !ok {
@@ -274,7 +248,7 @@ func TestVolumeWorkspaceReadsWaitForMutationLock(t *testing.T) {
}))
t.Cleanup(server.Close)
- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClient(t, server))
service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)
unlockMutation := service.workspaceLocks.Lock("workspace-volume")
readStarted := make(chan struct{})
@@ -344,7 +318,7 @@ func TestDownloadVolumeWorkspaceFileHoldsReadLockUntilClosed(t *testing.T) {
return
}
_, _ = output.Write(content)
- writeDockerExecAttachResponseInternal(t, w, output.String())
+ writeDockerExecAttachResponse(t, w, output.String())
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/exec/read-exec/json"):
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"Running": false, "ExitCode": 0})
@@ -354,7 +328,7 @@ func TestDownloadVolumeWorkspaceFileHoldsReadLockUntilClosed(t *testing.T) {
}))
t.Cleanup(server.Close)
- dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClientInternal(t, server))
+ dockerService := docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(newVolumeWorkspaceTestDockerClient(t, server))
service := NewVolumeService(nil, dockerService, nil, nil, nil, nil, nil, nil, nil, nil)
service.helperByVolume["workspace-volume"] = &volumeHelper{id: "helper", lastUsedAt: time.Now(), protocol: types.ProtocolVersion}
reader, size, err := service.workspace.DownloadVolumeWorkspaceFile(t.Context(), "workspace-volume", "file.txt")
@@ -407,7 +381,7 @@ func TestUpdateVolumeWorkspaceRejectsStaleRevisionBeforeStaging(t *testing.T) {
"{\"end\":true,\"version\":2}\n",
"{\"version\":\"0.2.0\",\"revision\":\"test\",\"buildTime\":\"test\",\"protocol\":2}\n",
)
- writeDockerExecAttachResponseInternal(t, w, output)
+ writeDockerExecAttachResponse(t, w, output)
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/exec/workspace-exec-") && strings.HasSuffix(r.URL.Path, "/json"):
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"Running": false, "ExitCode": 0})
@@ -420,7 +394,7 @@ func TestUpdateVolumeWorkspaceRejectsStaleRevisionBeforeStaging(t *testing.T) {
}))
t.Cleanup(server.Close)
- dockerClient := newVolumeWorkspaceTestDockerClientInternal(t, server)
+ dockerClient := newVolumeWorkspaceTestDockerClient(t, server)
service := NewVolumeService(nil, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), nil, nil, nil, nil, nil, nil, nil, nil)
uploadIndex := 0
workspace, err := service.workspace.UpdateVolumeWorkspace(t.Context(), "workspace-volume", volumetypes.WorkspaceUpdateManifest{
@@ -472,11 +446,11 @@ func TestVolumeBackup_ListResolvesDestinationName(t *testing.T) {
func TestVolumeBackupPolicy_GetReturnsLastRunForEachPolicyFromStore(t *testing.T) {
gormDB, err := gorm.Open(sqlite.Open("file:volume-backup-last-runs-store?mode=memory&cache=shared"), &gorm.Config{})
require.NoError(t, err)
- applyVolumeBackupMigrationsInternal(t, gormDB)
+ applyVolumeBackupMigrations(t, gormDB)
require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "first-policy", Status: VolumeBackupStatusSucceeded}).Error)
require.NoError(t, gormDB.Create(&VolumeBackup{VolumeName: "app-data", PolicyID: "second-policy", Status: VolumeBackupStatusFailed}).Error)
- store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStoreInternal()
+ store := NewVolumeService(&database.DB{DB: gormDB}, nil, nil, nil, nil, nil, nil, nil, nil, nil).backupStore()
first, err := store.Latest(t.Context(), "first-policy")
require.NoError(t, err)
require.Equal(t, "succeeded", first.Status)
diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.go
index eb74b5ae6..140793645 100644
--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go
+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go
@@ -325,7 +325,9 @@ func (s *Service) scanResult(
// The name can move to another image after it was selected; never store that image's findings under imageID.
// Compare layer diff IDs: image IDs differ by image store (containerd uses the manifest digest).
if mode == "exec" {
- inspect, inspectErr := dockerClient.ImageInspect(ctx, imageID)
+ inspect, inspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {
+ return cli.ImageInspect(ctx, imageID)
+ })
if inspectErr != nil {
return nil, fmt.Errorf("selected image %s is no longer available: %w", imageID, inspectErr)
}
@@ -387,25 +389,31 @@ func (s *Service) Version(ctx context.Context) string {
return ""
}
- resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{
- Config: &container.Config{
- Image: scannerImage,
- Entrypoint: []string{"trivy"},
- Cmd: []string{"--version"},
- Labels: map[string]string{
- libarcane.InternalResourceLabel: "true",
+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {
+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: &container.Config{
+ Image: scannerImage,
+ Entrypoint: []string{"trivy"},
+ Cmd: []string{"--version"},
+ Labels: map[string]string{
+ libarcane.InternalResourceLabel: "true",
+ },
},
- },
- HostConfig: &container.HostConfig{
- AutoRemove: true,
- },
+ HostConfig: &container.HostConfig{
+ AutoRemove: true,
+ },
+ })
})
if err != nil {
return ""
}
- if _, containerStartErr := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {
+ return cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {
+ return cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ })
return ""
}
@@ -415,7 +423,9 @@ func (s *Service) Version(ctx context.Context) string {
Follow: true,
})
if err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ _, _ = s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerRemoveResult, error) {
+ return cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})
+ })
return ""
}
var closeLogsOnce sync.Once
@@ -518,10 +528,6 @@ func (s *Service) resolveTrivyRuntimeOptions(ctx context.Context, dockerClient *
}, nil
}
-func (s *Service) scanTimeoutArg() string {
- return timeouts.GetDuration(s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()
-}
-
func (s *Service) EnsureImage(ctx context.Context) (string, error) {
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
@@ -529,7 +535,9 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {
}
scannerImage := volumehelper.ToolsImage(s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value)
- if _, imageInspectErr := dockerClient.ImageInspect(ctx, scannerImage); imageInspectErr == nil {
+ if _, imageInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {
+ return cli.ImageInspect(ctx, scannerImage)
+ }); imageInspectErr == nil {
return scannerImage, nil
}
@@ -553,20 +561,19 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {
}
func (s *Service) ensureTrivyCacheVolume(ctx context.Context) (string, error) {
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return "", err
- }
-
- if _, volumeInspectErr := dockerClient.VolumeInspect(ctx, trivyCacheVolumeName, client.VolumeInspectOptions{}); volumeInspectErr == nil {
+ if _, volumeInspectErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.VolumeInspectResult, error) {
+ return cli.VolumeInspect(ctx, trivyCacheVolumeName, client.VolumeInspectOptions{})
+ }); volumeInspectErr == nil {
return trivyCacheVolumeName, nil
}
- _, err = dockerClient.VolumeCreate(ctx, client.VolumeCreateOptions{
- Name: trivyCacheVolumeName,
- Labels: map[string]string{
- libarcane.InternalResourceLabel: "true",
- },
+ _, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.VolumeCreateResult, error) {
+ return cli.VolumeCreate(ctx, client.VolumeCreateOptions{
+ Name: trivyCacheVolumeName,
+ Labels: map[string]string{
+ libarcane.InternalResourceLabel: "true",
+ },
+ })
})
if err != nil {
return "", fmt.Errorf("failed to create trivy cache volume: %w", err)
@@ -614,8 +621,10 @@ func (s *Service) trivyContainerFiles(ctx context.Context) map[string][]byte {
// Flags passed here take precedence over the user-supplied trivy.yaml.
func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string, files map[string][]byte) []string {
serverURL, _ := s.ServerConfig()
- cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.scanTimeoutArg()}
- cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, s.settingsService.GetSettingsConfig().TrivyDbRegistry.Value)...)
+ cfg := s.settingsService.GetSettingsConfig()
+ scanTimeout := timeouts.GetDuration(cfg.TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()
+ cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", scanTimeout}
+ cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, cfg.TrivyDbRegistry.Value)...)
cmdArgs = append(cmdArgs, vuln.IgnoreUnfixedArgs(s.IgnoreUnfixed())...)
if files[vuln.TrivyConfigFile] != nil {
@@ -646,12 +655,11 @@ func (s *Service) createAndStartTrivyContainer(
) (string, error) {
vuln.LogContainerStartupRequest(ctx, scope, containerConfig, hostConfig)
- createCtx, createCancel := context.WithTimeout(ctx, timeouts.GetDuration(0, timeouts.DefaultDockerAPI))
- defer createCancel()
-
- resp, err := dockerClient.ContainerCreate(createCtx, client.ContainerCreateOptions{
- Config: containerConfig,
- HostConfig: hostConfig,
+ resp, err := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerCreateResult, error) {
+ return cli.ContainerCreate(ctx, client.ContainerCreateOptions{
+ Config: containerConfig,
+ HostConfig: hostConfig,
+ })
})
if err != nil {
slog.WarnContext(ctx, "failed to create trivy container", "scope", scope, "error", err)
@@ -672,10 +680,9 @@ func (s *Service) createAndStartTrivyContainer(
)
}
- startCtx, startCancel := context.WithTimeout(ctx, timeouts.GetDuration(0, timeouts.DefaultDockerAPI))
- defer startCancel()
-
- if _, containerStartErr := dockerClient.ContainerStart(startCtx, resp.ID, client.ContainerStartOptions{}); containerStartErr != nil {
+ if _, containerStartErr := s.dockerService.Call(ctx, func(ctx context.Context, cli *client.Client) (client.ContainerStartResult, error) {
+ return cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{})
+ }); containerStartErr != nil {
stdoutLog, stderrLog := vuln.ReadStartupLogs(ctx, dockerClient, resp.ID)
slog.WarnContext(ctx,
"failed to start trivy container",
@@ -902,7 +909,7 @@ func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID str
logDone := docker.StartStdCopy(logs, stdoutFile, stderrFile)
startTime := time.Now()
- waitCtx, waitCancel := context.WithTimeout(ctx, vuln.ContainerWaitTimeout(s.scanTimeoutArg()))
+ waitCtx, waitCancel := context.WithTimeout(ctx, vuln.ContainerWaitTimeout(timeouts.GetDuration(s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()))
defer waitCancel()
waitResult := dockerClient.ContainerWait(waitCtx, containerID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
diff --git a/backend/internal/vulnerability/children/scan/service.go b/backend/internal/vulnerability/children/scan/service.go
index 3b5b92242..06030c735 100644
--- a/backend/internal/vulnerability/children/scan/service.go
+++ b/backend/internal/vulnerability/children/scan/service.go
@@ -31,7 +31,6 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/scan/children/trivy"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
- "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
@@ -176,11 +175,9 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user use
}
scanCtx := utils.ActivityRuntimeContext(ctx, nil)
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return nil, fmt.Errorf("failed to connect to Docker: %w", err)
- }
- imageInspect, err := dockerClient.ImageInspect(scanCtx, imageID)
+ imageInspect, err := s.dockerService.Call(scanCtx, func(ctx context.Context, cli *client.Client) (client.ImageInspectResult, error) {
+ return cli.ImageInspect(ctx, imageID)
+ })
if err != nil {
return nil, fmt.Errorf("failed to inspect image: %w", err)
}
@@ -556,12 +553,10 @@ func (s *Service) ScanTarget(ctx context.Context, envID string, target Target) (
defer func() {
// A cancel only detaches from the exec, so the container is restarted to stop Trivy before another scan reuses it.
if ctx.Err() != nil {
- restartCtx, cancelRestart := context.WithTimeout(context.WithoutCancel(ctx), timeouts.GetDuration(0, timeouts.DefaultDockerAPI))
- defer cancelRestart()
- dockerClient, clientErr := s.dockerService.GetClient(restartCtx)
- if clientErr == nil {
- _, clientErr = dockerClient.ContainerRestart(restartCtx, containerID, client.ContainerRestartOptions{})
- }
+ restartCtx := context.WithoutCancel(ctx)
+ _, clientErr := s.dockerService.Call(restartCtx, func(ctx context.Context, cli *client.Client) (client.ContainerRestartResult, error) {
+ return cli.ContainerRestart(ctx, containerID, client.ContainerRestartOptions{})
+ })
if clientErr != nil {
// Without this container the pool cannot serve scans, so later scans start a fresh one.
slog.WarnContext(restartCtx, "scheduled vulnerability scan: failed to stop a canceled scan; retiring its scan containers", "containerId", containerID, "error", clientErr)
diff --git a/backend/internal/vulnerability/children/store/service.go b/backend/internal/vulnerability/children/store/service.go
index 7a2d961ca..59374fe8c 100644
--- a/backend/internal/vulnerability/children/store/service.go
+++ b/backend/internal/vulnerability/children/store/service.go
@@ -213,12 +213,6 @@ func (s *Service) ImportLegacyReportFiles(ctx context.Context) {
}
}
-// DeleteScanResult deletes the scan result for an image
-func (s *Service) DeleteScanResult(ctx context.Context, imageID string) error {
- s.deleteScanChildRows(ctx, []string{imageID})
- return s.records.Delete(ctx, []string{imageID})
-}
-
// DeleteScanResultsByImageIDs deletes scan results for multiple images in one query.
func (s *Service) DeleteScanResultsByImageIDs(ctx context.Context, imageIDs []string) error {
if len(imageIDs) == 0 {
diff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.go
index 34d0eac4d..13ce3b9bf 100644
--- a/backend/internal/vulnerability/service.go
+++ b/backend/internal/vulnerability/service.go
@@ -450,11 +450,6 @@ func (s *VulnerabilityService) SyncThreatIntel(ctx context.Context, force bool)
return s.store.SyncThreatIntel(ctx, force)
}
-// DeleteScanResult deletes the scan result for an image.
-func (s *VulnerabilityService) DeleteScanResult(ctx context.Context, imageID string) error {
- return s.store.DeleteScanResult(ctx, imageID)
-}
-
// DeleteScanResultsByImageIDs deletes scan results for multiple images.
func (s *VulnerabilityService) DeleteScanResultsByImageIDs(ctx context.Context, imageIDs []string) error {
return s.store.DeleteScanResultsByImageIDs(ctx, imageIDs)
diff --git a/backend/pkg/libarcane/startup/runtime_identity.go b/backend/pkg/libarcane/startup/runtime_identity.go
index 8fd607e2d..822d217e1 100644
--- a/backend/pkg/libarcane/startup/runtime_identity.go
+++ b/backend/pkg/libarcane/startup/runtime_identity.go
@@ -20,7 +20,7 @@ import (
const (
defaultDataDirectory = "/app/data"
defaultBuildsDirectory = "/builds"
- defaultDatabaseURL = "file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate"
+ defaultDatabaseURL = "file:data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate"
defaultDockerConfigDir = "/app/data/.docker"
defaultDockerSocketPath = "/var/run/docker.sock"
defaultRuntimeUID = 65532
@@ -66,8 +66,8 @@ func ApplyRequestedRuntimeIdentity(ctx context.Context, cfg *RuntimeIdentityConf
projectsDir = filepath.Clean(strings.TrimSpace(cfg.ProjectsDirectory))
}
- inContainer := runningInContainerInternal(os.Getenv, os.Stat)
- req, warning, err := loadRuntimeIdentityRequestInternal(cfg, inContainer)
+ inContainer := runningInContainer(os.Getenv, os.Stat)
+ req, warning, err := loadRuntimeIdentityRequest(cfg, inContainer)
if warning != "" {
slog.WarnContext(ctx, "Runtime identity warning", "warning", warning)
}
@@ -75,50 +75,37 @@ func ApplyRequestedRuntimeIdentity(ctx context.Context, cfg *RuntimeIdentityConf
return err
}
if !req.Enabled {
- return ensureSQLiteFilesExistInternal(cfg.DatabaseURL)
+ return ensureSQLiteFilesExist(cfg.DatabaseURL)
}
- runtimeUID := req.UID
- runtimeGID := req.GID
-
- // Avoid re-execing forever when the requested runtime identity is already active.
- if os.Geteuid() == runtimeUID && os.Getegid() == runtimeGID {
- if ensureRuntimeDockerConfigErr := ensureRuntimeDockerConfigInternal(cfg, os.Setenv, runtimeUID, runtimeGID, inContainer); ensureRuntimeDockerConfigErr != nil {
- return ensureRuntimeDockerConfigErr
- }
- return ensureSQLiteFilesExistInternal(cfg.DatabaseURL)
+ if configErr := ensureRuntimeDockerConfig(cfg, os.Setenv, req.UID, req.GID, inContainer); configErr != nil {
+ return configErr
}
- if os.Geteuid() != 0 {
+ switch {
+ case os.Geteuid() == req.UID && os.Getegid() == req.GID:
+ // Avoid re-execing forever when the requested runtime identity is already active.
+ return ensureSQLiteFilesExist(cfg.DatabaseURL)
+ case os.Geteuid() != 0:
slog.WarnContext(ctx, "Runtime identity warning: process is not root, continuing as current user",
- "euid", os.Geteuid(), "puid", runtimeUID, "pgid", runtimeGID)
- if ensureRuntimeDockerConfigErr2 := ensureRuntimeDockerConfigInternal(cfg, os.Setenv, runtimeUID, runtimeGID, inContainer); ensureRuntimeDockerConfigErr2 != nil {
- return ensureRuntimeDockerConfigErr2
- }
- return ensureSQLiteFilesExistInternal(cfg.DatabaseURL)
- }
-
- if ensureRuntimeDockerConfigErr3 := ensureRuntimeDockerConfigInternal(cfg, os.Setenv, runtimeUID, runtimeGID, inContainer); ensureRuntimeDockerConfigErr3 != nil {
- return ensureRuntimeDockerConfigErr3
+ "euid", os.Geteuid(), "puid", req.UID, "pgid", req.GID)
+ return ensureSQLiteFilesExist(cfg.DatabaseURL)
}
if inContainer {
- mountpoints, loadMountpointsErr := loadMountpointsInternal(mountInfoPath)
- if loadMountpointsErr != nil {
- return fmt.Errorf("load mountpoints: %w", loadMountpointsErr)
+ mountInfo, readErr := os.ReadFile(mountInfoPath)
+ if readErr != nil && !os.IsNotExist(readErr) {
+ return fmt.Errorf("load mountpoints: %w", readErr)
}
-
- if prepareWritablePathsWithRootsErr := prepareWritablePathsWithRootsInternal(
- runtimeUID, runtimeGID, mountpoints, projectsDir, defaultDataDirectory, defaultBuildsDirectory,
- ); prepareWritablePathsWithRootsErr != nil {
- return prepareWritablePathsWithRootsErr
+ if prepareErr := prepareWritablePathsWithRoots(req.UID, req.GID, parseMountpoints(string(mountInfo)), projectsDir, defaultDataDirectory, defaultBuildsDirectory); prepareErr != nil {
+ return prepareErr
}
}
- return reexecWithRuntimeIdentityInternal(ctx, req)
+ return reexecWithRuntimeIdentity(ctx, req)
}
-func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainer bool) (runtimeIdentityRequest, string, error) {
+func loadRuntimeIdentityRequest(cfg *RuntimeIdentityConfig, inContainer bool) (runtimeIdentityRequest, string, error) {
if cfg == nil {
cfg = &RuntimeIdentityConfig{}
}
@@ -126,12 +113,22 @@ func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainer
puid := strings.TrimSpace(cfg.PUID)
pgid := strings.TrimSpace(cfg.PGID)
- if puid == "" && pgid == "" {
- return defaultRuntimeIdentityRequestInternal(cfg.DockerHost, inContainer), "", nil
- }
-
if puid == "" || pgid == "" {
- req := defaultRuntimeIdentityRequestInternal(cfg.DockerHost, inContainer)
+ // Without both overrides, containers run as the default non-root user.
+ req := runtimeIdentityRequest{DockerHost: cfg.DockerHost}
+ if inContainer {
+ req = runtimeIdentityRequest{
+ Enabled: true,
+ UID: defaultRuntimeUID,
+ GID: defaultRuntimeGID,
+ CredentialUID: uint32(defaultRuntimeUID),
+ CredentialGID: uint32(defaultRuntimeGID),
+ DockerHost: cfg.DockerHost,
+ }
+ }
+ if puid == "" && pgid == "" {
+ return req, "", nil
+ }
warning := kit.Ternary(
inContainer,
"PUID and PGID must both be set to override the default non-root runtime user; continuing with the default non-root runtime user",
@@ -140,12 +137,12 @@ func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainer
return req, warning, nil
}
- uid, credentialUID, err := parseRuntimeIdentityValueInternal(puid, "PUID")
+ uid, credentialUID, err := parseRuntimeIdentityValue(puid, "PUID")
if err != nil {
return runtimeIdentityRequest{}, "", fmt.Errorf("invalid PUID %q: %w", puid, err)
}
- gid, credentialGID, err := parseRuntimeIdentityValueInternal(pgid, "PGID")
+ gid, credentialGID, err := parseRuntimeIdentityValue(pgid, "PGID")
if err != nil {
return runtimeIdentityRequest{}, "", fmt.Errorf("invalid PGID %q: %w", pgid, err)
}
@@ -160,25 +157,7 @@ func loadRuntimeIdentityRequestInternal(cfg *RuntimeIdentityConfig, inContainer
}, "", nil
}
-func defaultRuntimeIdentityRequestInternal(dockerHost string, inContainer bool) runtimeIdentityRequest {
- if !inContainer {
- return runtimeIdentityRequest{
- Enabled: false,
- DockerHost: dockerHost,
- }
- }
-
- return runtimeIdentityRequest{
- Enabled: true,
- UID: defaultRuntimeUID,
- GID: defaultRuntimeGID,
- CredentialUID: uint32(defaultRuntimeUID),
- CredentialGID: uint32(defaultRuntimeGID),
- DockerHost: dockerHost,
- }
-}
-
-func runningInContainerInternal(getenv func(string) string, stat func(string) (os.FileInfo, error)) bool {
+func runningInContainer(getenv func(string) string, stat func(string) (os.FileInfo, error)) bool {
if kit.ParseOrDefault(strings.TrimSpace(getenv("ARCANE_IN_CONTAINER")), false, strconv.ParseBool) {
return true
}
@@ -196,8 +175,8 @@ func runningInContainerInternal(getenv func(string) string, stat func(string) (o
return false
}
-func ensureRuntimeDockerConfigInternal(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) error {
- configDir, err := configureRuntimeDockerConfigEnvInternal(cfg, setenv, uid, gid, inContainer)
+func ensureRuntimeDockerConfig(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) error {
+ configDir, err := configureRuntimeDockerConfigEnv(cfg, setenv, uid, gid, inContainer)
if err != nil {
return err
}
@@ -218,7 +197,7 @@ func ensureRuntimeDockerConfigInternal(cfg *RuntimeIdentityConfig, setenv func(s
return nil
}
-func configureRuntimeDockerConfigEnvInternal(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) (string, error) {
+func configureRuntimeDockerConfigEnv(cfg *RuntimeIdentityConfig, setenv func(string, string) error, uid, gid int, inContainer bool) (string, error) {
if cfg == nil {
cfg = &RuntimeIdentityConfig{}
}
@@ -246,51 +225,28 @@ func configureRuntimeDockerConfigEnvInternal(cfg *RuntimeIdentityConfig, setenv
return configDir, nil
}
-func runtimeIdentitySupplementaryGroupsInternal(dockerHost string, resolveSocketGroup func(string) mo.Option[uint32]) []uint32 {
- socketPath, ok := dockerSocketPathInternal(dockerHost).Get()
- if !ok {
- return nil
+func runtimeIdentitySupplementaryGroups(dockerHost string, socketGroup func(string) mo.Option[uint32]) []uint32 {
+ socketPath := defaultDockerSocketPath
+ if value := strings.TrimSpace(dockerHost); value != "" {
+ parsed, err := url.Parse(value)
+ if err != nil || parsed.Scheme != "unix" {
+ return nil
+ }
+ socketPath = cmp.Or(parsed.Host+parsed.Path, parsed.Opaque)
+ if socketPath == "" {
+ return nil
+ }
+ socketPath = filepath.Clean("/" + strings.TrimPrefix(socketPath, "/"))
}
- socketGID, ok := resolveSocketGroup(socketPath).Get()
+ socketGID, ok := socketGroup(socketPath).Get()
if !ok {
return nil
}
return []uint32{socketGID}
}
-func dockerSocketPathInternal(raw string) mo.Option[string] {
- value := strings.TrimSpace(raw)
- if value == "" {
- return mo.Some(defaultDockerSocketPath)
- }
-
- parsed, err := url.Parse(value)
- if err != nil || parsed.Scheme != "unix" {
- return mo.None[string]()
- }
-
- if parsed.Host != "" || parsed.Path != "" {
- socketPath := parsed.Host + parsed.Path
- if !strings.HasPrefix(socketPath, "/") {
- socketPath = "/" + socketPath
- }
- return mo.Some(filepath.Clean(socketPath))
- }
-
- if parsed.Opaque == "" {
- return mo.None[string]()
- }
-
- socketPath := strings.TrimPrefix(parsed.Opaque, "//")
- if !strings.HasPrefix(socketPath, "/") {
- socketPath = "/" + socketPath
- }
-
- return mo.Some(filepath.Clean(socketPath))
-}
-
-func prepareWritablePathsWithRootsInternal(uid, gid int, mountpoints map[string]struct{}, projectsDir, dataDirectory, buildsDirectory string) error {
+func prepareWritablePathsWithRoots(uid, gid int, mountpoints map[string]struct{}, projectsDir, dataDirectory, buildsDirectory string) error {
if err := os.MkdirAll(dataDirectory, utils.DirPerm); err != nil {
return fmt.Errorf("create data directory: %w", err)
}
@@ -318,7 +274,7 @@ func prepareWritablePathsWithRootsInternal(uid, gid int, mountpoints map[string]
}
continue
}
- if chownRecursiveErr := chownRecursiveInternal(entryPath, uid, gid, mountpoints, projectsDir); chownRecursiveErr != nil {
+ if chownRecursiveErr := chownRecursive(entryPath, uid, gid, mountpoints, projectsDir); chownRecursiveErr != nil {
return fmt.Errorf("chown %s: %w", entryPath, chownRecursiveErr)
}
}
@@ -337,15 +293,15 @@ func prepareWritablePathsWithRootsInternal(uid, gid int, mountpoints map[string]
return fmt.Errorf("stat builds directory: %w", statErr)
}
- if chownRecursiveErr2 := chownRecursiveInternal(buildsDirectory, uid, gid, mountpoints, projectsDir); chownRecursiveErr2 != nil {
+ if chownRecursiveErr2 := chownRecursive(buildsDirectory, uid, gid, mountpoints, projectsDir); chownRecursiveErr2 != nil {
return fmt.Errorf("chown builds directory: %w", chownRecursiveErr2)
}
return nil
}
-func ensureSQLiteFilesExistInternal(databaseURL string) error {
- sqlitePath, ok, err := sqliteDatabasePathInternal(databaseURL)
+func ensureSQLiteFilesExist(databaseURL string) error {
+ sqlitePath, ok, err := sqliteDatabasePath(databaseURL)
if err != nil {
return err
}
@@ -355,7 +311,7 @@ func ensureSQLiteFilesExistInternal(databaseURL string) error {
// Ensure the parent directory exists before creating the file.
// This covers the "already the right user" early-return path where
- // prepareWritablePathsWithRootsInternal is not called.
+ // prepareWritablePathsWithRoots is not called.
dir := filepath.Dir(sqlitePath)
if dir != "" && dir != "." {
if mkdirAllErr := os.MkdirAll(dir, utils.DirPerm); mkdirAllErr != nil {
@@ -374,28 +330,16 @@ func ensureSQLiteFilesExistInternal(databaseURL string) error {
return nil
}
-func sqliteDatabasePathInternal(databaseURL string) (string, bool, error) {
+func sqliteDatabasePath(databaseURL string) (string, bool, error) {
value := cmp.Or(strings.TrimSpace(databaseURL), defaultDatabaseURL)
if !strings.HasPrefix(value, "file:") {
return "", false, nil
}
- parsed, err := url.Parse(value)
+ pathPart, err := kit.SQLitePathFromDSN(value)
if err != nil {
return "", false, fmt.Errorf("parse sqlite database url: %w", err)
}
-
- // For relative URLs like "file:data/arcane.db", url.Parse puts the path in
- // Opaque (without a leading slash). For absolute URLs like "file:/app/data/arcane.db",
- // Opaque is empty and Path contains the absolute path. Only strip the leading
- // slash from the opaque portion to preserve absolute paths.
- var pathPart string
- if parsed.Opaque != "" {
- pathPart = strings.TrimPrefix(parsed.Opaque, "/")
- } else {
- pathPart = parsed.Path
- }
-
if pathPart == "" || strings.HasPrefix(pathPart, ":memory:") {
return "", false, nil
}
@@ -403,7 +347,7 @@ func sqliteDatabasePathInternal(databaseURL string) (string, bool, error) {
return filepath.Clean(pathPart), true, nil
}
-func chownRecursiveInternal(path string, uid, gid int, mountpoints map[string]struct{}, projectsDir string) error {
+func chownRecursive(path string, uid, gid int, mountpoints map[string]struct{}, projectsDir string) error {
return filepath.Walk(path, func(currentPath string, info os.FileInfo, err error) error {
if err != nil {
return err
@@ -425,18 +369,7 @@ func chownRecursiveInternal(path string, uid, gid int, mountpoints map[string]st
})
}
-func loadMountpointsInternal(path string) (map[string]struct{}, error) {
- data, err := os.ReadFile(path)
- if err != nil {
- if os.IsNotExist(err) {
- return map[string]struct{}{}, nil
- }
- return nil, err
- }
- return parseMountpointsInternal(string(data)), nil
-}
-
-func parseMountpointsInternal(data string) map[string]struct{} {
+func parseMountpoints(data string) map[string]struct{} {
mountpoints := make(map[string]struct{})
for line := range strings.SplitSeq(data, "\n") {
@@ -450,17 +383,17 @@ func parseMountpointsInternal(data string) map[string]struct{} {
continue
}
- mountpoint := filepath.Clean(unescapeMountInfoPathInternal(fields[4]))
+ mountpoint := filepath.Clean(unescapeMountInfoPath(fields[4]))
mountpoints[mountpoint] = struct{}{}
}
return mountpoints
}
-// unescapeMountInfoPathInternal decodes the kernel's octal escape sequences
+// unescapeMountInfoPath decodes the kernel's octal escape sequences
// used in /proc/self/mountinfo. The kernel only uses \040 (space), \011 (tab),
// \012 (newline), and \134 (backslash) — no other escape forms appear.
-func unescapeMountInfoPathInternal(path string) string {
+func unescapeMountInfoPath(path string) string {
replacer := strings.NewReplacer(
`\040`, " ",
`\011`, "\t",
diff --git a/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go b/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go
index 8ca6eed95..c0b68628c 100644
--- a/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go
+++ b/backend/pkg/libarcane/startup/runtime_identity_int_32bit.go
@@ -7,7 +7,7 @@ import (
"strconv"
)
-func parseRuntimeIdentityValueInternal(raw, key string) (int, uint32, error) {
+func parseRuntimeIdentityValue(raw, key string) (int, uint32, error) {
value, err := strconv.ParseInt(raw, 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("parse %s: %w", key, err)
diff --git a/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go b/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go
index 9bc3015e3..c748d1ba4 100644
--- a/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go
+++ b/backend/pkg/libarcane/startup/runtime_identity_int_64bit.go
@@ -8,7 +8,7 @@ import (
"strconv"
)
-func parseRuntimeIdentityValueInternal(raw, key string) (int, uint32, error) {
+func parseRuntimeIdentityValue(raw, key string) (int, uint32, error) {
value, err := strconv.Atoi(raw)
if err != nil {
return 0, 0, fmt.Errorf("parse %s: %w", key, err)
diff --git a/backend/pkg/libarcane/startup/runtime_identity_linux.go b/backend/pkg/libarcane/startup/runtime_identity_linux.go
index 87a6161f3..47c05648b 100644
--- a/backend/pkg/libarcane/startup/runtime_identity_linux.go
+++ b/backend/pkg/libarcane/startup/runtime_identity_linux.go
@@ -13,13 +13,13 @@ import (
"github.com/samber/mo"
)
-func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityRequest) error {
+func reexecWithRuntimeIdentity(ctx context.Context, req runtimeIdentityRequest) error {
executable, err := os.Executable()
if err != nil {
return fmt.Errorf("resolve executable: %w", err)
}
- groups := runtimeIdentitySupplementaryGroupsInternal(req.DockerHost, resolveSocketGroupInternal)
+ groups := runtimeIdentitySupplementaryGroups(req.DockerHost, resolveSocketGroup)
cmd := exec.CommandContext(ctx, executable, os.Args[1:]...) //nolint:gosec // re-executing our own binary with the same args under a different UID/GID
// Forward shutdown instead of the default SIGKILL so the child can unregister its actor host.
@@ -59,7 +59,7 @@ func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityR
return fmt.Errorf("wait for runtime identity child: %w", waitErr)
}
-func resolveSocketGroupInternal(socketPath string) mo.Option[uint32] {
+func resolveSocketGroup(socketPath string) mo.Option[uint32] {
// os.* rather than acfs: the docker socket is a host-configured system path
// probed during identity bootstrap, not under any acfs root.
info, err := os.Stat(socketPath)
diff --git a/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go b/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go
index 2146784c7..97f54e094 100644
--- a/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go
+++ b/backend/pkg/libarcane/startup/runtime_identity_nonlinux.go
@@ -8,11 +8,8 @@ import (
"runtime"
)
-var (
- _ = runtimeIdentitySupplementaryGroupsInternal
- _ = dockerSocketPathInternal
-)
+var _ = runtimeIdentitySupplementaryGroups
-func reexecWithRuntimeIdentityInternal(_ context.Context, req runtimeIdentityRequest) error {
+func reexecWithRuntimeIdentity(_ context.Context, req runtimeIdentityRequest) error {
return fmt.Errorf("runtime identity switching is only supported on linux, current platform is %s", runtime.GOOS)
}
diff --git a/backend/pkg/libarcane/startup/runtime_identity_test.go b/backend/pkg/libarcane/startup/runtime_identity_test.go
index 6e3846294..cc8973631 100644
--- a/backend/pkg/libarcane/startup/runtime_identity_test.go
+++ b/backend/pkg/libarcane/startup/runtime_identity_test.go
@@ -14,7 +14,7 @@ import (
func TestLoadRuntimeIdentityRequest(t *testing.T) {
t.Run("disabled when unset outside a container", func(t *testing.T) {
- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{
+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{
DockerHost: "unix:///tmp/docker.sock",
}, false)
require.NoError(t, err)
@@ -24,7 +24,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {
})
t.Run("enabled with default non-root runtime when unset in container", func(t *testing.T) {
- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{
+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{
DockerHost: "unix:///tmp/docker.sock",
}, true)
require.NoError(t, err)
@@ -38,7 +38,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {
})
t.Run("warning when partial config", func(t *testing.T) {
- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "1001"}, true)
+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "1001"}, true)
require.NoError(t, err)
require.Contains(t, warning, "PUID and PGID must both be set")
require.True(t, req.Enabled)
@@ -49,7 +49,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {
})
t.Run("warning when partial config outside container", func(t *testing.T) {
- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "1001"}, false)
+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "1001"}, false)
require.NoError(t, err)
require.Contains(t, warning, "PUID and PGID must both be set to enable runtime identity outside containers")
require.False(t, req.Enabled)
@@ -60,7 +60,7 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {
})
t.Run("error when invalid numeric value", func(t *testing.T) {
- _, _, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "abc", PGID: "1001"}, false)
+ _, _, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "abc", PGID: "1001"}, false)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid PUID")
})
@@ -68,13 +68,13 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {
t.Run("error when value exceeds uint32", func(t *testing.T) {
tooLarge := strconv.FormatUint(uint64(math.MaxUint32)+1, 10)
- _, _, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: tooLarge, PGID: "1001"}, false)
+ _, _, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: tooLarge, PGID: "1001"}, false)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid PUID")
})
t.Run("enabled when both set", func(t *testing.T) {
- req, warning, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{
+ req, warning, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{
PUID: "1001",
PGID: "2001",
DockerHost: "unix:///tmp/docker.sock",
@@ -90,15 +90,15 @@ func TestLoadRuntimeIdentityRequest(t *testing.T) {
})
t.Run("error when value is negative", func(t *testing.T) {
- _, _, err := loadRuntimeIdentityRequestInternal(&RuntimeIdentityConfig{PUID: "-1", PGID: "1001"}, false)
+ _, _, err := loadRuntimeIdentityRequest(&RuntimeIdentityConfig{PUID: "-1", PGID: "1001"}, false)
require.Error(t, err)
require.Contains(t, err.Error(), "invalid PUID")
})
}
-func TestRunningInContainerInternal(t *testing.T) {
+func TestRunningInContainer(t *testing.T) {
t.Run("explicit arcane env enables container mode", func(t *testing.T) {
- inContainer := runningInContainerInternal(
+ inContainer := runningInContainer(
func(key string) string {
return kit.Ternary(key == "ARCANE_IN_CONTAINER", "true", "")
},
@@ -111,7 +111,7 @@ func TestRunningInContainerInternal(t *testing.T) {
})
t.Run("container env enables container mode", func(t *testing.T) {
- inContainer := runningInContainerInternal(
+ inContainer := runningInContainer(
func(key string) string {
return kit.Ternary(key == "container", "podman", "")
},
@@ -124,7 +124,7 @@ func TestRunningInContainerInternal(t *testing.T) {
})
t.Run("container marker file enables container mode", func(t *testing.T) {
- inContainer := runningInContainerInternal(
+ inContainer := runningInContainer(
func(string) string { return "" },
func(path string) (os.FileInfo, error) {
return nil, kit.Ternary(path == "/run/.containerenv", nil, os.ErrNotExist)
@@ -135,7 +135,7 @@ func TestRunningInContainerInternal(t *testing.T) {
})
t.Run("no signal disables container mode", func(t *testing.T) {
- inContainer := runningInContainerInternal(
+ inContainer := runningInContainer(
func(string) string { return "" },
func(string) (os.FileInfo, error) {
return nil, os.ErrNotExist
@@ -151,7 +151,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {
cfg := &RuntimeIdentityConfig{}
env := map[string]string{}
- configDir, err := configureRuntimeDockerConfigEnvInternal(
+ configDir, err := configureRuntimeDockerConfigEnv(
cfg,
func(key, value string) error {
env[key] = value
@@ -173,7 +173,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {
env := map[string]string{}
setCalled := false
- configDir, err := configureRuntimeDockerConfigEnvInternal(
+ configDir, err := configureRuntimeDockerConfigEnv(
cfg,
func(key, value string) error {
setCalled = true
@@ -197,7 +197,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {
env := map[string]string{}
setCalled := false
- configDir, err := configureRuntimeDockerConfigEnvInternal(
+ configDir, err := configureRuntimeDockerConfigEnv(
cfg,
func(key, value string) error {
setCalled = true
@@ -221,7 +221,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {
env := map[string]string{}
setCalled := false
- configDir, err := configureRuntimeDockerConfigEnvInternal(
+ configDir, err := configureRuntimeDockerConfigEnv(
cfg,
func(key, value string) error {
setCalled = true
@@ -244,7 +244,7 @@ func TestConfigureRuntimeDockerConfigEnv(t *testing.T) {
cfg := &RuntimeIdentityConfig{}
env := map[string]string{}
- configDir, err := configureRuntimeDockerConfigEnvInternal(
+ configDir, err := configureRuntimeDockerConfigEnv(
cfg,
func(key, value string) error {
env[key] = value
@@ -266,7 +266,7 @@ func TestEnsureRuntimeDockerConfigCreatesCustomDirectory(t *testing.T) {
customDir := filepath.Join(t.TempDir(), "custom-docker-config")
cfg := &RuntimeIdentityConfig{DockerConfig: customDir}
- require.NoError(t, ensureRuntimeDockerConfigInternal(
+ require.NoError(t, ensureRuntimeDockerConfig(
cfg,
func(string, string) error {
require.FailNow(t, "setenv should not be called for explicit DOCKER_CONFIG")
@@ -285,7 +285,7 @@ func TestEnsureRuntimeDockerConfigCreatesCustomDirectory(t *testing.T) {
func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {
t.Run("maps default docker socket group when docker host unset", func(t *testing.T) {
- groups := runtimeIdentitySupplementaryGroupsInternal(
+ groups := runtimeIdentitySupplementaryGroups(
"",
func(socketPath string) mo.Option[uint32] {
require.Equal(t, defaultDockerSocketPath, socketPath)
@@ -297,7 +297,7 @@ func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {
})
t.Run("maps custom unix docker host socket group", func(t *testing.T) {
- groups := runtimeIdentitySupplementaryGroupsInternal(
+ groups := runtimeIdentitySupplementaryGroups(
"unix:///tmp/docker.sock",
func(socketPath string) mo.Option[uint32] {
require.Equal(t, "/tmp/docker.sock", socketPath)
@@ -311,7 +311,7 @@ func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {
t.Run("skips non unix docker host", func(t *testing.T) {
called := false
- groups := runtimeIdentitySupplementaryGroupsInternal(
+ groups := runtimeIdentitySupplementaryGroups(
"tcp://docker:2375",
func(string) mo.Option[uint32] {
called = true
@@ -324,7 +324,7 @@ func TestRuntimeIdentitySupplementaryGroups(t *testing.T) {
})
t.Run("skips socket group when socket lookup fails", func(t *testing.T) {
- groups := runtimeIdentitySupplementaryGroupsInternal(
+ groups := runtimeIdentitySupplementaryGroups(
"unix:///tmp/missing.sock",
func(string) mo.Option[uint32] { return mo.None[uint32]() },
)
@@ -340,7 +340,7 @@ func TestParseMountpoints(t *testing.T) {
99 92 0:46 / /builds rw,relatime - ext4 /dev/sdc1 rw
`
- parsed := parseMountpointsInternal(data)
+ parsed := parseMountpoints(data)
require.Contains(t, parsed, "/app/data")
require.Contains(t, parsed, "/app/data/projects")
require.Contains(t, parsed, "/builds")
@@ -348,39 +348,39 @@ func TestParseMountpoints(t *testing.T) {
func TestSQLiteDatabasePath(t *testing.T) {
t.Run("uses default sqlite path when unset", func(t *testing.T) {
- path, ok, err := sqliteDatabasePathInternal("")
+ path, ok, err := sqliteDatabasePath("")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "data/arcane.db", path)
})
t.Run("preserves absolute sqlite path", func(t *testing.T) {
- path, ok, err := sqliteDatabasePathInternal("file:/app/custom/arcane.db?_pragma=journal_mode(WAL)")
+ path, ok, err := sqliteDatabasePath("file:/app/custom/arcane.db?_pragma=journal_mode(WAL)")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "/app/custom/arcane.db", path)
})
t.Run("returns relative sqlite path", func(t *testing.T) {
- path, ok, err := sqliteDatabasePathInternal("file:data/arcane.db?_pragma=journal_mode(WAL)")
+ path, ok, err := sqliteDatabasePath("file:data/arcane.db?_pragma=journal_mode(WAL)")
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "data/arcane.db", path)
})
t.Run("skips non sqlite database urls", func(t *testing.T) {
- path, ok, err := sqliteDatabasePathInternal("postgres://arcane:secret@db/arcane")
+ path, ok, err := sqliteDatabasePath("postgres://arcane:secret@db/arcane")
require.NoError(t, err)
require.False(t, ok)
require.Empty(t, path)
})
}
-func TestEnsureSQLiteFilesExistInternal(t *testing.T) {
+func TestEnsureSQLiteFilesExist(t *testing.T) {
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "arcane.db")
- require.NoError(t, ensureSQLiteFilesExistInternal("file:"+dbPath))
+ require.NoError(t, ensureSQLiteFilesExist("file:"+dbPath))
require.FileExists(t, dbPath)
require.NoFileExists(t, dbPath+"-wal")
@@ -391,7 +391,7 @@ func TestEnsureSQLiteFilesExistInternalCreatesParentDir(t *testing.T) {
tempDir := t.TempDir()
dbPath := filepath.Join(tempDir, "subdir", "arcane.db")
- require.NoError(t, ensureSQLiteFilesExistInternal("file:"+dbPath))
+ require.NoError(t, ensureSQLiteFilesExist("file:"+dbPath))
require.FileExists(t, dbPath)
}
@@ -411,7 +411,7 @@ func TestUnescapeMountInfoPath(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- require.Equal(t, tt.expect, unescapeMountInfoPathInternal(tt.input))
+ require.Equal(t, tt.expect, unescapeMountInfoPath(tt.input))
})
}
}
@@ -446,7 +446,7 @@ func TestChownRecursiveInternalSkipsProjects(t *testing.T) {
})
mountpoints := make(map[string]struct{})
- err := chownRecursiveInternal(tempDir, 1000, 1000, mountpoints, filepath.Clean(projectsDir))
+ err := chownRecursive(tempDir, 1000, 1000, mountpoints, filepath.Clean(projectsDir))
require.NoError(t, err)
cleanedTempDir := filepath.Clean(tempDir)
@@ -490,7 +490,7 @@ func TestPrepareWritablePathsInternalChownsTopLevelProjectsShallowly(t *testing.
lchownFn = oldLchownFn
})
- err := prepareWritablePathsWithRootsInternal(os.Getuid(), os.Getgid(), map[string]struct{}{}, filepath.Clean(projectsDir), dataDir, buildsDir)
+ err := prepareWritablePathsWithRoots(os.Getuid(), os.Getgid(), map[string]struct{}{}, filepath.Clean(projectsDir), dataDir, buildsDir)
require.NoError(t, err)
require.True(t, visited[filepath.Clean(projectsDir)], "top-level projects dir must be chowned")
diff --git a/backend/pkg/scheduler/runs/activity.go b/backend/pkg/scheduler/runs/activity.go
index 0b007c936..3bdd04d89 100644
--- a/backend/pkg/scheduler/runs/activity.go
+++ b/backend/pkg/scheduler/runs/activity.go
@@ -13,7 +13,7 @@ import (
// SetObserver connects activity projection before accepting work.
func (q *Coordinator) SetObserver(observer scheduler.RunObserver) { q.observer = observer }
-func (q *Coordinator) associateActivityInternal(run *scheduler.Run) {
+func (q *Coordinator) associateActivity(run *scheduler.Run) {
if q.observer != nil && run.ActivityID == "" {
run.ActivityID = q.observer.ActivityID(*run)
}
@@ -22,22 +22,22 @@ func (q *Coordinator) associateActivityInternal(run *scheduler.Run) {
}
}
-func (q *Coordinator) observeRunInternal(ctx context.Context, run scheduler.Run) {
+func (q *Coordinator) observeRun(ctx context.Context, run scheduler.Run) {
if q.observer == nil {
return
}
q.observerMu.Lock()
defer q.observerMu.Unlock()
- q.syncActivityInternal(ctx, run)
- q.signalInternal()
+ q.syncActivity(ctx, run)
+ q.signal()
}
-func (q *Coordinator) syncActivityInternal(ctx context.Context, run scheduler.Run) {
+func (q *Coordinator) syncActivity(ctx context.Context, run scheduler.Run) {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
var state scheduler.CoordinatorState
id := kit.SHA256Hex(run.EnvironmentID + "\x00" + run.JobID)
- if err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state); err != nil {
+ if err := q.service.GetState(ctx, coordinatorType, id, &state); err != nil {
return
}
version, pending := state.PendingActivitySync[run.ID]
@@ -49,20 +49,20 @@ func (q *Coordinator) syncActivityInternal(ctx context.Context, run scheduler.Ru
err = q.observer.SyncRunActivity(ctx, current)
}
if err == nil {
- _, err = q.service.Invoke(ctx, coordinatorTypeInternal, id, "activity-synced", scheduler.ActivitySyncCommand{RunID: run.ID, UpdatedAt: version})
+ _, err = q.service.Invoke(ctx, coordinatorType, id, "activity-synced", scheduler.ActivitySyncCommand{RunID: run.ID, UpdatedAt: version})
}
if err != nil {
slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)
}
}
-func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {
+func (q *Coordinator) retryActivitySync(ctx context.Context) {
if q.observer == nil {
return
}
cursor := ""
for {
- page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
if err != nil {
slog.ErrorContext(ctx, "List pending job activities", "error", err)
return
@@ -78,7 +78,7 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {
return
}
q.observerMu.Lock()
- q.syncActivityInternal(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})
+ q.syncActivity(ctx, scheduler.Run{ID: runID, EnvironmentID: state.Record.EnvironmentID, JobID: state.Record.JobID})
q.observerMu.Unlock()
}
}
@@ -89,12 +89,12 @@ func (q *Coordinator) retryActivitySyncInternal(ctx context.Context) {
}
}
-func (q *Coordinator) hasPendingActivitySyncInternal(ctx context.Context) bool {
+func (q *Coordinator) hasPendingActivitySync(ctx context.Context) bool {
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
cursor := ""
for {
- page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
if err != nil {
return true
}
diff --git a/backend/pkg/scheduler/runs/activity_test.go b/backend/pkg/scheduler/runs/activity_test.go
index aad9002c9..8113b4c1c 100644
--- a/backend/pkg/scheduler/runs/activity_test.go
+++ b/backend/pkg/scheduler/runs/activity_test.go
@@ -11,20 +11,20 @@ import (
"github.com/stretchr/testify/require"
)
-type activityObserverFakeInternal struct {
+type activityObserverFake struct {
unavailable atomic.Bool
observed atomic.Int32
existingOnly bool
}
-func (o *activityObserverFakeInternal) ActivityID(run st.Run) string {
+func (o *activityObserverFake) ActivityID(run st.Run) string {
if o.existingOnly {
return run.ActivityID
}
return run.ID
}
-func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run) error {
+func (o *activityObserverFake) SyncRunActivity(context.Context, st.Run) error {
if o.unavailable.Load() {
return errors.New("activity storage unavailable")
}
@@ -33,8 +33,8 @@ func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run)
}
func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
- q := newResolutionQueueInternal(t)
- observer := &activityObserverFakeInternal{}
+ q := newResolutionQueue(t)
+ observer := &activityObserverFake{}
observer.unavailable.Store(true)
q.SetObserver(observer)
var executions atomic.Int32
@@ -56,10 +56,10 @@ func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {
persisted, getErr := q.Get(t.Context(), "0", run.JobID, run.ID)
return getErr == nil && persisted.Status == st.Succeeded
}, 5*time.Second, 10*time.Millisecond)
- require.True(t, q.hasPendingActivitySyncInternal(t.Context()))
+ require.True(t, q.hasPendingActivitySync(t.Context()))
observer.unavailable.Store(false)
- q.retryActivitySyncInternal(t.Context())
- require.False(t, q.hasPendingActivitySyncInternal(t.Context()))
+ q.retryActivitySync(t.Context())
+ require.False(t, q.hasPendingActivitySync(t.Context()))
require.Positive(t, observer.observed.Load())
duplicate, err := q.Submit(t.Context(), st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "manual"})
require.NoError(t, err)
diff --git a/backend/pkg/scheduler/runs/actors.go b/backend/pkg/scheduler/runs/actors.go
index 1f6d33a41..1201c95a7 100644
--- a/backend/pkg/scheduler/runs/actors.go
+++ b/backend/pkg/scheduler/runs/actors.go
@@ -1,6 +1,7 @@
package runs
import (
+ "cmp"
"context"
"encoding/json/v2"
"errors"
@@ -22,51 +23,51 @@ import (
)
const (
- coordinatorTypeInternal = "job-coordinator"
- executorTypeInternal = "job-executor"
- healthExecutorTypeInternal = "job-health-executor"
- orphanedDispatchAge = 15 * time.Minute
- outcomePersistGrace = 30 * time.Second
+ coordinatorType = "job-coordinator"
+ executorType = "job-executor"
+ healthExecutorType = "job-health-executor"
+ orphanedDispatchAge = 15 * time.Minute
+ outcomePersistGrace = 30 * time.Second
)
-type coordinatorActorInternal struct {
+type coordinatorActor struct {
id string
q *Coordinator
}
-type executorActorInternal struct {
+type executorActor struct {
q *Coordinator
}
func (q *Coordinator) Register(runtime *francis.Runtime) error {
q.runtime = runtime
- if err := runtime.RegisterActor(coordinatorTypeInternal, func(id string, _ *actor.Service) actor.Actor {
- return &coordinatorActorInternal{id: id, q: q}
+ if err := runtime.RegisterActor(coordinatorType, func(id string, _ *actor.Service) actor.Actor {
+ return &coordinatorActor{id: id, q: q}
}); err != nil {
return err
}
executorFactory := func(_ string, _ *actor.Service) actor.Actor {
- return &executorActorInternal{q: q}
+ return &executorActor{q: q}
}
if err := runtime.RegisterActor(
- executorTypeInternal, executorFactory,
+ executorType, executorFactory,
local.WithCapacityGroup("jobs", 4),
local.WithCompletedJobRetention(7*24*time.Hour),
); err != nil {
return err
}
return runtime.RegisterActor(
- healthExecutorTypeInternal, executorFactory,
+ healthExecutorType, executorFactory,
local.WithCapacityGroup("jobs-health", 2),
local.WithCompletedJobRetention(7*24*time.Hour),
)
}
-func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {
+func (a *coordinatorActor) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {
var state st.CoordinatorState
- if err := a.q.service.GetState(ctx, coordinatorTypeInternal, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ if err := a.q.service.GetState(ctx, coordinatorType, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
return nil, err
}
- normalizeRecordTimesInternal(&state.Record)
+ normalizeRecordTimes(&state.Record)
switch method {
case "replace":
var next st.CoordinatorState
@@ -77,13 +78,13 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, da
return false, nil
}
next.Revision++
- if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, next, nil); err != nil {
+ if err := a.q.service.SetState(ctx, coordinatorType, a.id, next, nil); err != nil {
return nil, err
}
// Acceptance is persisted; dispatch failures are retried.
- if err := a.flushInternal(ctx, &next); err != nil {
+ if err := a.flush(ctx, &next); err != nil {
slog.WarnContext(ctx, "Job dispatch requires reconciliation", "jobId", next.Record.JobID, "environmentId", next.Record.EnvironmentID, "error", err)
- a.q.signalInternal()
+ a.q.signal()
}
return true, nil
case "activity-synced":
@@ -97,9 +98,9 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, da
}
delete(state.PendingActivitySync, command.RunID)
state.Revision++
- return nil, a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil)
+ return nil, a.q.service.SetState(ctx, coordinatorType, a.id, state, nil)
case "repair":
- return nil, a.flushInternal(ctx, &state)
+ return nil, a.flush(ctx, &state)
case "import":
var command st.CoordinatorImport
if err := data.Decode(&command); err != nil {
@@ -117,17 +118,17 @@ func (a *coordinatorActorInternal) Invoke(ctx context.Context, method string, da
state.ImportHash = command.SourceHash
state.Record = command.Record
for _, run := range state.Record.Runs {
- markActivitySyncInternal(&state, run)
+ markActivitySync(&state, run)
}
state.Imported = true
state.Revision = 1
- return true, a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil)
+ return true, a.q.service.SetState(ctx, coordinatorType, a.id, state, nil)
default:
return nil, fmt.Errorf("unknown coordinator command: %s", method)
}
}
-func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.CoordinatorState) error {
+func (a *coordinatorActor) flush(ctx context.Context, state *st.CoordinatorState) error {
if !a.q.enabled.Load() {
return nil
}
@@ -138,7 +139,7 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.
DueTime: record.NextRun,
Data: st.ScheduleOccurrence{Generation: record.Generation, Sequence: record.Sequence},
}
- if err := a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, properties); err != nil {
+ if err := a.q.service.SetAlarm(ctx, coordinatorType, a.id, name, properties); err != nil {
return err
}
}
@@ -151,14 +152,14 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.
}
var retryAt time.Time
if selected == nil {
- selected, retryAt = a.q.nextRunInternal(record.Runs)
+ selected, retryAt = a.q.nextRun(record.Runs)
}
if selected == nil {
if retryAt.IsZero() {
return nil
}
name := fmt.Sprintf("retry-%d", retryAt.UnixNano())
- return a.q.service.SetAlarm(ctx, coordinatorTypeInternal, a.id, name, actor.AlarmProperties{DueTime: retryAt})
+ return a.q.service.SetAlarm(ctx, coordinatorType, a.id, name, actor.AlarmProperties{DueTime: retryAt})
}
if selected.Status == st.Running {
active, err := a.reconcileDispatch(ctx, state, *selected)
@@ -173,7 +174,7 @@ func (a *coordinatorActorInternal) flushInternal(ctx context.Context, state *st.
}
// reconcileDispatch recovers a lost job ID before considering redelivery.
-func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {
+func (a *coordinatorActor) reconcileDispatch(ctx context.Context, state *st.CoordinatorState, run st.Run) (bool, error) {
key := fmt.Sprintf("%s-%d", run.ID, run.AttemptCount)
previous, exists := state.Dispatches[key]
if !exists {
@@ -181,8 +182,8 @@ func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state
}
actorType := kit.Ternary(
strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",
- healthExecutorTypeInternal,
- executorTypeInternal,
+ healthExecutorType,
+ executorType,
)
if previous.JobID == "" {
command := st.ExecutionCommand{
@@ -198,7 +199,7 @@ func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state
previous.JobID = id
state.Dispatches[key] = previous
state.Revision++
- if setStateErr := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil); setStateErr != nil {
+ if setStateErr := a.q.service.SetState(ctx, coordinatorType, a.id, *state, nil); setStateErr != nil {
return false, setStateErr
}
}
@@ -219,15 +220,15 @@ func (a *coordinatorActorInternal) reconcileDispatch(ctx context.Context, state
return false, nil
}
-func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {
+func (a *coordinatorActor) dispatchRun(ctx context.Context, state *st.CoordinatorState, run st.Run, attempt int) error {
key := fmt.Sprintf("%s-%d", run.ID, attempt)
if state.Dispatches == nil {
state.Dispatches = make(map[string]st.DispatchIntent)
}
actorType := kit.Ternary(
strings.HasPrefix(run.JobID, "environment-health") || run.JobID == "docker-client-refresh" || run.JobID == "activity-sweep",
- healthExecutorTypeInternal,
- executorTypeInternal,
+ healthExecutorType,
+ executorType,
)
intent := state.Dispatches[key]
if intent.JobID != "" {
@@ -244,10 +245,9 @@ func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.Co
}
}
}
- intent = st.DispatchIntent{RunID: run.ID, Attempt: attempt}
- state.Dispatches[key] = intent
+ state.Dispatches[key] = st.DispatchIntent{RunID: run.ID, Attempt: attempt}
state.Revision++
- if err := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil); err != nil {
+ if err := a.q.service.SetState(ctx, coordinatorType, a.id, *state, nil); err != nil {
return err
}
options := []actor.JobOption{actor.WithIdempotencyKey(key)}
@@ -277,23 +277,22 @@ func (a *coordinatorActorInternal) dispatchRun(ctx context.Context, state *st.Co
return err
}
}
- intent.JobID = id
- state.Dispatches[key] = intent
+ state.Dispatches[key] = st.DispatchIntent{RunID: run.ID, Attempt: attempt, JobID: id}
state.Revision++
- return a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, *state, nil)
+ return a.q.service.SetState(ctx, coordinatorType, a.id, *state, nil)
}
-func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data actor.Envelope) error {
+func (a *coordinatorActor) Alarm(ctx context.Context, name string, data actor.Envelope) error {
if !a.q.enabled.Load() {
return nil
}
var state st.CoordinatorState
- if err := a.q.service.GetState(ctx, coordinatorTypeInternal, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
+ if err := a.q.service.GetState(ctx, coordinatorType, a.id, &state); err != nil && !errors.Is(err, actor.ErrStateNotFound) {
return err
}
- normalizeRecordTimesInternal(&state.Record)
+ normalizeRecordTimes(&state.Record)
if strings.HasPrefix(name, "retry-") {
- return a.flushInternal(ctx, &state)
+ return a.flush(ctx, &state)
}
var occurrence st.ScheduleOccurrence
if err := data.Decode(&occurrence); err != nil {
@@ -319,17 +318,9 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data
if !nextOccurrence.After(now) {
trigger = "recovery"
}
- pending := false
- for _, run := range record.Runs {
- if separateTriggerInternal(run.Trigger) {
- continue
- }
- if run.Status.Terminal() || run.Status == st.NeedsAttention {
- continue
- }
- pending = true
- break
- }
+ pending := slices.ContainsFunc(record.Runs, func(run st.Run) bool {
+ return !separateTrigger(run.Trigger) && !run.Status.Terminal() && run.Status != st.NeedsAttention
+ })
if !pending {
run := st.Run{
ID: uuid.New().String(),
@@ -340,23 +331,23 @@ func (a *coordinatorActorInternal) Alarm(ctx context.Context, name string, data
CreatedAt: now,
UpdatedAt: now,
}
- a.q.associateActivityInternal(&run)
+ a.q.associateActivity(&run)
record.Runs = append(record.Runs, run)
- markActivitySyncInternal(&state, run)
+ markActivitySync(&state, run)
}
record.LastEnqueuedAt = now
record.NextRun = schedule.Next(now.In(a.q.location))
record.Sequence++
state.Revision++
- if setStateErr := a.q.service.SetState(ctx, coordinatorTypeInternal, a.id, state, nil); setStateErr != nil {
+ if setStateErr := a.q.service.SetState(ctx, coordinatorType, a.id, state, nil); setStateErr != nil {
return setStateErr
}
- return a.flushInternal(ctx, &state)
+ return a.flush(ctx, &state)
}
-// repairInternal flushes coordinators so schedule alarms and dispatches are restored.
+// repair flushes coordinators so schedule alarms and dispatches are restored.
// Unless all is set, it skips records without unfinished runs so idle coordinators can deactivate.
-func (q *Coordinator) repairInternal(ctx context.Context, all bool) error {
+func (q *Coordinator) repair(ctx context.Context, all bool) error {
records, err := q.Records(ctx)
if err != nil {
return err
@@ -365,15 +356,15 @@ func (q *Coordinator) repairInternal(ctx context.Context, all bool) error {
if !all && !slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention }) {
continue
}
- if _, invokeErr := q.service.Invoke(ctx, coordinatorTypeInternal, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {
+ if _, invokeErr := q.service.Invoke(ctx, coordinatorType, kit.SHA256Hex(record.EnvironmentID+"\x00"+record.JobID), "repair", nil); invokeErr != nil {
return invokeErr
}
}
return nil
}
-func (q *Coordinator) importLegacyInternal(ctx context.Context) error {
- entries, err := q.store.ListByPrefix(ctx, queuePrefixInternal)
+func (q *Coordinator) importLegacy(ctx context.Context) error {
+ entries, err := q.store.ListByPrefix(ctx, queuePrefix)
if err != nil {
return fmt.Errorf("list legacy job records: %w", err)
}
@@ -391,16 +382,16 @@ func (q *Coordinator) importLegacyInternal(ctx context.Context) error {
return fmt.Errorf("decode legacy job record %q: %w", entry.Key, unmarshalErr)
}
id := kit.SHA256Hex(record.EnvironmentID + "\x00" + record.JobID)
- if entry.Key != queuePrefixInternal+id {
+ if entry.Key != queuePrefix+id {
return fmt.Errorf("invalid legacy job key %q for job %q in environment %q", entry.Key, record.JobID, record.EnvironmentID)
}
sourceHash := kit.SHA256Hex(entry.Value)
command := st.CoordinatorImport{Record: record, SourceHash: sourceHash}
- if _, invokeErr := q.service.Invoke(ctx, coordinatorTypeInternal, id, "import", command); invokeErr != nil {
+ if _, invokeErr := q.service.Invoke(ctx, coordinatorType, id, "import", command); invokeErr != nil {
return fmt.Errorf("import legacy job record %q: %w", entry.Key, invokeErr)
}
var imported st.CoordinatorState
- if getStateErr := q.service.GetState(ctx, coordinatorTypeInternal, id, &imported); getStateErr != nil {
+ if getStateErr := q.service.GetState(ctx, coordinatorType, id, &imported); getStateErr != nil {
return fmt.Errorf("import legacy job record %q: %w", entry.Key, getStateErr)
}
if !imported.Imported || imported.ImportHash != sourceHash {
@@ -416,7 +407,7 @@ func (q *Coordinator) importLegacyInternal(ctx context.Context) error {
return nil
}
-func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.Envelope) error {
+func (a *executorActor) Job(ctx context.Context, _ string, data actor.Envelope) error {
q := a.q
q.mu.Lock()
lifetime := q.lifecycleCtx
@@ -438,7 +429,7 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.En
if err := data.Decode(&command); err != nil {
return errors.Join(err, actor.ErrJobPermanentFailure)
}
- claim, err := q.claimExecutionInternal(ctx, command)
+ claim, err := q.claimExecution(ctx, command)
if err != nil {
return err
}
@@ -448,38 +439,43 @@ func (a *executorActorInternal) Job(ctx context.Context, _ string, data actor.En
if !claim.Claimed {
return nil
}
- outcome, runErr := q.invokeInternal(ctx, claim.Run, claim.Reconcile)
- if outcome.Status == "" {
- if runErr != nil {
- outcome.Status = st.Failed
- } else {
- outcome.Status = st.Succeeded
+ outcome, runErr := func() (outcome st.Outcome, err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ outcome = st.Outcome{Status: st.Failed, Message: "Job panicked; inspect logs before retrying"}
+ err = fmt.Errorf("job panic: %v", recovered)
+ slog.ErrorContext(ctx, "Job panicked", "runId", claim.Run.ID, "error", err)
+ }
+ }()
+ if claim.Reconcile && q.reconcile != nil {
+ return q.reconcile(ctx, claim.Run)
}
- }
+ return q.execute(ctx, claim.Run)
+ }()
// Domain recovery may still report uncertainty; job runs finish as failures.
- if outcome.Status == st.NeedsAttention {
- outcome.Status = st.Failed
- }
- if runErr != nil && outcome.Message == "" {
- outcome.Message = runErr.Error()
+ status := cmp.Or(outcome.Status, kit.Ternary(runErr != nil, st.Failed, st.Succeeded))
+ outcome = st.Outcome{
+ Status: kit.Ternary(status == st.NeedsAttention, st.Failed, status),
+ Message: cmp.Or(outcome.Message, kit.Ternary(runErr != nil, fmt.Sprint(runErr), "")),
+ ActivityID: outcome.ActivityID,
+ Targets: outcome.Targets,
+ Steps: outcome.Steps,
}
if runErr != nil {
if ctx.Err() != nil {
return ctx.Err()
}
- q.persistOutcome(ctx, claim.Run, outcome)
- return ctx.Err()
+ return cmp.Or(q.persistOutcome(ctx, claim.Run, outcome), ctx.Err())
}
persistCtx, cancel := context.WithCancel(context.WithoutCancel(ctx))
defer cancel()
defer context.AfterFunc(ctx, func() { time.AfterFunc(outcomePersistGrace, cancel) })()
- q.persistOutcome(persistCtx, claim.Run, outcome)
- return ctx.Err()
+ return cmp.Or(q.persistOutcome(persistCtx, claim.Run, outcome), ctx.Err())
}
-func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {
+func (q *Coordinator) claimExecution(ctx context.Context, command st.ExecutionCommand) (st.ExecutionClaim, error) {
var claim st.ExecutionClaim
- err := q.mutateInternal(ctx, command.EnvironmentID, command.JobID, func(record *st.QueueRecord) error {
+ err := q.mutate(ctx, command.EnvironmentID, command.JobID, func(record *st.QueueRecord) error {
claim = st.ExecutionClaim{}
for i := range record.Runs {
current := &record.Runs[i]
@@ -498,10 +494,10 @@ func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.Exe
// Executor turns serialize attempts for this coordinator.
now := time.Now().UTC()
if current.NextAttempt != nil && current.NextAttempt.After(now) {
- claim.Deferred = true
+ claim = st.ExecutionClaim{Deferred: true}
return nil
}
- claim.Reconcile = current.Status == st.Running
+ reconcile := current.Status == st.Running
current.Status = st.Running
current.Owner = q.owner + "/" + uuid.New().String()
current.UpdatedAt = now
@@ -515,8 +511,11 @@ func (q *Coordinator) claimExecutionInternal(ctx context.Context, command st.Exe
if len(current.Attempts) > 100 {
current.Attempts = current.Attempts[len(current.Attempts)-100:]
}
- claim.Run = *current
- claim.Claimed = true
+ claim = st.ExecutionClaim{
+ Run: *current,
+ Claimed: true,
+ Reconcile: reconcile,
+ }
return nil
}
return nil
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index d3f406091..44513a751 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -69,20 +69,20 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
}
data, marshalErr := json.Marshal(record)
require.NoError(t, marshalErr)
- require.NoError(t, store.Set(t.Context(), queuePrefixInternal+kit.SHA256Hex("0\x00"+jobID), string(data)))
+ require.NoError(t, store.Set(t.Context(), queuePrefix+kit.SHA256Hex("0\x00"+jobID), string(data)))
}
encoded, err := json.Marshal(legacy)
require.NoError(t, err)
- key := queuePrefixInternal + kit.SHA256Hex("0\x00"+legacy.JobID)
+ key := queuePrefix + kit.SHA256Hex("0\x00"+legacy.JobID)
require.NoError(t, store.Set(t.Context(), key, string(encoded)))
databaseURL := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "actors.db"))
first := francistest.New(t, databaseURL)
q := New(store, first.Service(), time.UTC)
require.NoError(t, q.Register(first))
francistest.Start(t, first)
- require.NoError(t, q.importLegacyInternal(t.Context()))
- require.NoError(t, q.importLegacyInternal(t.Context()))
- observer := &activityObserverFakeInternal{existingOnly: true}
+ require.NoError(t, q.importLegacy(t.Context()))
+ require.NoError(t, q.importLegacy(t.Context()))
+ observer := &activityObserverFake{existingOnly: true}
observer.unavailable.Store(true)
q.SetObserver(observer)
_, err = q.Submit(t.Context(), st.Request{JobID: legacy.JobID, RunID: legacy.Runs[0].ID, Trigger: "manual"})
@@ -91,7 +91,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
run.Outcome.Targets = []st.TargetOutcome{{ID: "artifact", Status: st.Succeeded, RecoveryData: jsontext.Value(`{"backupId":"artifact"}`)}}
return nil
}))
- require.True(t, q.hasPendingActivitySyncInternal(t.Context()))
+ require.True(t, q.hasPendingActivitySync(t.Context()))
q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
t.Error("startup must not execute jobs before activation")
return st.Outcome{Status: st.Failed}, nil
@@ -106,7 +106,7 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
restored.SetExecutor(q.execute, nil)
require.NoError(t, restored.Register(second))
francistest.Start(t, second)
- require.NoError(t, restored.importLegacyInternal(t.Context()))
+ require.NoError(t, restored.importLegacy(t.Context()))
require.NoError(t, restored.Start(t.Context(), t.Context()))
t.Cleanup(func() { require.NoError(t, restored.Stop(context.WithoutCancel(t.Context()))) })
for _, jobID := range []string{"gitops-sync:first", "gitops-sync:second"} {
@@ -128,31 +128,31 @@ func TestLegacyImportSurvivesRestartAndDeduplicates(t *testing.T) {
require.Equal(t, 1, history.Total)
require.Equal(t, time.UTC, history.Runs[0].CreatedAt.Location())
observer.unavailable.Store(false)
- restored.retryActivitySyncInternal(t.Context())
- require.False(t, restored.hasPendingActivitySyncInternal(t.Context()))
+ restored.retryActivitySync(t.Context())
+ require.False(t, restored.hasPendingActivitySync(t.Context()))
require.Positive(t, observer.observed.Load())
_, exists, err := store.Get(t.Context(), key)
require.NoError(t, err)
require.True(t, exists)
}
-type occurrenceEnvelopeInternal struct{ occurrence st.ScheduleOccurrence }
+type occurrenceEnvelope struct{ occurrence st.ScheduleOccurrence }
-func (e occurrenceEnvelopeInternal) Decode(value any) error {
+func (e occurrenceEnvelope) Decode(value any) error {
*value.(*st.ScheduleOccurrence) = e.occurrence
return nil
}
func TestAlarmIgnoresStaleGeneration(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
next := time.Now().UTC().Add(time.Hour)
require.NoError(t, q.Checkpoint(t.Context(), "schedule", "0 * * * * *", next))
before, err := q.ScheduleState(t.Context(), "schedule")
require.NoError(t, err)
q.enabled.Store(true)
defer q.enabled.Store(false)
- coordinator := &coordinatorActorInternal{id: kit.SHA256Hex("0\x00schedule"), q: q}
- require.NoError(t, coordinator.Alarm(t.Context(), "old", occurrenceEnvelopeInternal{st.ScheduleOccurrence{Generation: before.Generation - 1, Sequence: before.Sequence}}))
+ coordinator := &coordinatorActor{id: kit.SHA256Hex("0\x00schedule"), q: q}
+ require.NoError(t, coordinator.Alarm(t.Context(), "old", occurrenceEnvelope{st.ScheduleOccurrence{Generation: before.Generation - 1, Sequence: before.Sequence}}))
after, err := q.ScheduleState(t.Context(), "schedule")
require.NoError(t, err)
require.Equal(t, before.Generation, after.Generation)
@@ -168,7 +168,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
name = "canceled"
}
t.Run(name, func(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
var executed atomic.Int32
q.SetExecutor(func(context.Context, st.Run) (st.Outcome, error) {
executed.Add(1)
@@ -178,10 +178,10 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
require.NoError(t, err)
id := kit.SHA256Hex("0\x00" + run.JobID)
command := st.ExecutionCommand{EnvironmentID: "0", JobID: run.JobID, RunID: run.ID, Attempt: 1}
- jobID, created, err := q.service.Dispatch(t.Context(), executorTypeInternal, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))
+ jobID, created, err := q.service.Dispatch(t.Context(), executorType, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))
require.NoError(t, err)
require.True(t, created)
- duplicateID, created, err := q.service.Dispatch(t.Context(), executorTypeInternal, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))
+ duplicateID, created, err := q.service.Dispatch(t.Context(), executorType, id, "execute", command, actor.WithIdempotencyKey(run.ID+"-1"))
require.NoError(t, err)
require.False(t, created)
require.Equal(t, jobID, duplicateID)
@@ -215,7 +215,7 @@ func TestDuplicateDispatchAndCanceledDelivery(t *testing.T) {
}
func TestCapacityGroupsReserveHealthSlots(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
var ordinary, health atomic.Int32
release := make(chan struct{})
var once sync.Once
@@ -261,19 +261,19 @@ func TestCapacityGroupsReserveHealthSlots(t *testing.T) {
}
func TestReconciliationRepairsMissingAlarm(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
require.NoError(t, q.Checkpoint(t.Context(), "repair", "0 * * * * *", time.Now().UTC().Add(time.Hour)))
q.enabled.Store(true)
defer q.enabled.Store(false)
record, err := q.ScheduleState(t.Context(), "repair")
require.NoError(t, err)
id := kit.SHA256Hex("0\x00repair")
- err = q.repairInternal(t.Context(), true)
+ err = q.repair(t.Context(), true)
require.NoError(t, err)
- require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
- err = q.repairInternal(t.Context(), true)
+ require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
+ err = q.repair(t.Context(), true)
require.NoError(t, err)
- require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorTypeInternal, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
+ require.NoError(t, q.service.DeleteAlarm(t.Context(), coordinatorType, id, fmt.Sprintf("schedule-%d-%d", record.Generation, record.Sequence)))
after, err := q.ScheduleState(t.Context(), "repair")
require.NoError(t, err)
require.Equal(t, record.Generation, after.Generation)
@@ -307,12 +307,12 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {
require.True(t, admitted)
replacement.Release(t.Context())
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
run, err := q.Submit(t.Context(), st.Request{JobID: "tokens"})
require.NoError(t, err)
require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error { current.Status = st.Running; current.Owner = "new-token"; return nil }))
run.Owner = "stale-token"
- q.persistOutcome(t.Context(), run, st.Outcome{Status: st.Succeeded})
+ require.NoError(t, q.persistOutcome(t.Context(), run, st.Outcome{Status: st.Succeeded}))
persisted, err := q.Get(t.Context(), "0", run.JobID, run.ID)
require.NoError(t, err)
require.Equal(t, st.Running, persisted.Status)
@@ -320,7 +320,7 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {
}
func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
entered := make(chan struct{})
finish := make(chan struct{})
var once sync.Once
@@ -349,7 +349,7 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
}
id := kit.SHA256Hex("0\x00" + run.JobID)
var state st.CoordinatorState
- require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))
+ require.NoError(t, q.service.GetState(t.Context(), coordinatorType, id, &state))
key := run.ID + "-1"
original := state.Dispatches[key].JobID
require.NotEmpty(t, original)
@@ -357,12 +357,12 @@ func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
intent.JobID = ""
state.Dispatches[key] = intent
state.Revision++
- require.NoError(t, q.service.SetState(t.Context(), coordinatorTypeInternal, id, state, nil))
- err = q.repairInternal(t.Context(), false)
+ require.NoError(t, q.service.SetState(t.Context(), coordinatorType, id, state, nil))
+ err = q.repair(t.Context(), false)
require.NoError(t, err)
- require.NoError(t, q.service.GetState(t.Context(), coordinatorTypeInternal, id, &state))
+ require.NoError(t, q.service.GetState(t.Context(), coordinatorType, id, &state))
require.Equal(t, original, state.Dispatches[key].JobID)
- deliveries, err := q.service.ListJobs(t.Context(), executorTypeInternal, id)
+ deliveries, err := q.service.ListJobs(t.Context(), executorType, id)
require.NoError(t, err)
require.Len(t, deliveries, 1)
once.Do(func() { close(finish) })
diff --git a/backend/pkg/scheduler/runs/admission.go b/backend/pkg/scheduler/runs/admission.go
index d8da20767..73a2d9158 100644
--- a/backend/pkg/scheduler/runs/admission.go
+++ b/backend/pkg/scheduler/runs/admission.go
@@ -15,7 +15,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis"
)
-const admissionTypeInternal = "resource-admission"
+const admissionType = "resource-admission"
type Admission struct {
service *actor.Service
@@ -29,7 +29,7 @@ type Lease struct {
token string
once sync.Once
}
-type admissionActorInternal struct {
+type admissionActor struct {
id string
service *actor.Service
}
@@ -39,8 +39,8 @@ func NewAdmission(service *actor.Service, epoch string) *Admission {
}
func (a *Admission) Register(runtime *francis.Runtime) error {
- return runtime.RegisterActor(admissionTypeInternal, func(id string, service *actor.Service) actor.Actor {
- return &admissionActorInternal{id: id, service: service}
+ return runtime.RegisterActor(admissionType, func(id string, service *actor.Service) actor.Actor {
+ return &admissionActor{id: id, service: service}
})
}
@@ -50,20 +50,20 @@ func (a *Admission) TryAcquire(ctx context.Context, key st.AdmissionKey) (*Lease
if err := ctx.Err(); err != nil {
return nil, false, err
}
- if err := a.retryReleasesInternal(ctx, id); err != nil {
+ if err := a.retryReleases(ctx, id); err != nil {
return nil, false, err
}
// Finish admission after submission so cancellation cannot orphan a lease.
invokeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
defer cancel()
- envelope, err := a.service.Invoke(invokeCtx, admissionTypeInternal, id, "acquire", st.AdmissionCommand{Epoch: a.epoch, Token: token})
+ envelope, err := a.service.Invoke(invokeCtx, admissionType, id, "acquire", st.AdmissionCommand{Epoch: a.epoch, Token: token})
if err != nil {
- a.releaseInternal(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})
+ a.release(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})
return nil, false, err
}
var acquired bool
if decodeErr := envelope.Decode(&acquired); decodeErr != nil {
- a.releaseInternal(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})
+ a.release(ctx, id, st.AdmissionCommand{Epoch: a.epoch, Token: token})
return nil, false, decodeErr
}
if !acquired {
@@ -77,11 +77,11 @@ func (l *Lease) Release(ctx context.Context) {
return
}
l.once.Do(func() {
- l.admission.releaseInternal(ctx, l.id, st.AdmissionCommand{Epoch: l.admission.epoch, Token: l.token})
+ l.admission.release(ctx, l.id, st.AdmissionCommand{Epoch: l.admission.epoch, Token: l.token})
})
}
-func (a *Admission) releaseInternal(ctx context.Context, id string, command st.AdmissionCommand) {
+func (a *Admission) release(ctx context.Context, id string, command st.AdmissionCommand) {
a.mu.Lock()
if a.pending[id] == nil {
a.pending[id] = make(map[string]st.AdmissionCommand)
@@ -90,12 +90,12 @@ func (a *Admission) releaseInternal(ctx context.Context, id string, command st.A
a.mu.Unlock()
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
defer cancel()
- if err := a.retryReleasesInternal(ctx, id); err != nil {
+ if err := a.retryReleases(ctx, id); err != nil {
slog.ErrorContext(ctx, "Resource admission release failed; will retry before further admission", "resource", id, "error", err)
}
}
-func (a *Admission) retryReleasesInternal(ctx context.Context, id string) error {
+func (a *Admission) retryReleases(ctx context.Context, id string) error {
a.mu.Lock()
pending := make([]st.AdmissionCommand, 0, len(a.pending[id]))
for _, command := range a.pending[id] {
@@ -103,7 +103,7 @@ func (a *Admission) retryReleasesInternal(ctx context.Context, id string) error
}
a.mu.Unlock()
for _, command := range pending {
- if _, err := a.service.Invoke(ctx, admissionTypeInternal, id, "release", command); err != nil {
+ if _, err := a.service.Invoke(ctx, admissionType, id, "release", command); err != nil {
return err
}
a.mu.Lock()
@@ -116,9 +116,9 @@ func (a *Admission) retryReleasesInternal(ctx context.Context, id string) error
return nil
}
-func (a *admissionActorInternal) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {
+func (a *admissionActor) Invoke(ctx context.Context, method string, data actor.Envelope) (any, error) {
var state st.AdmissionState
- err := a.service.GetState(ctx, admissionTypeInternal, a.id, &state)
+ err := a.service.GetState(ctx, admissionType, a.id, &state)
if err != nil && !errors.Is(err, actor.ErrStateNotFound) {
return nil, err
}
@@ -140,7 +140,7 @@ func (a *admissionActorInternal) Invoke(ctx context.Context, method string, data
default:
return nil, errors.New("unknown resource admission command")
}
- if setStateErr := a.service.SetState(ctx, admissionTypeInternal, a.id, state, nil); setStateErr != nil {
+ if setStateErr := a.service.SetState(ctx, admissionType, a.id, state, nil); setStateErr != nil {
return nil, setStateErr
}
return true, nil
diff --git a/backend/pkg/scheduler/runs/coordinator.go b/backend/pkg/scheduler/runs/coordinator.go
index 1242a90b7..a97c668f4 100644
--- a/backend/pkg/scheduler/runs/coordinator.go
+++ b/backend/pkg/scheduler/runs/coordinator.go
@@ -63,9 +63,7 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, e
return st.Run{}, errors.New("job ID required")
}
request.Trigger = cmp.Or(request.Trigger, "manual")
- if request.RunID == "" {
- request.RunID = uuid.New().String()
- }
+ request.RunID = cmp.Or(request.RunID, uuid.New().String())
if _, err := uuid.Parse(request.RunID); err != nil {
return st.Run{}, errors.New("invalid run ID")
}
@@ -84,8 +82,13 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, e
return st.Run{}, errors.New("observed agent run is not awaiting legacy resolution")
}
observed = *agentRun
- remoteOutcome = new(observed.Outcome)
- remoteOutcome.Status = observed.Status
+ remoteOutcome = &st.Outcome{
+ Status: observed.Status,
+ Message: observed.Outcome.Message,
+ ActivityID: observed.Outcome.ActivityID,
+ Targets: observed.Outcome.Targets,
+ Steps: observed.Outcome.Steps,
+ }
}
result := st.Run{
ID: request.RunID,
@@ -106,13 +109,16 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, e
RemoteAttemptCount: observed.AttemptCount,
RemoteOutcome: remoteOutcome,
}
- q.associateActivityInternal(&result)
- err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {
+ q.associateActivity(&result)
+ err := q.mutate(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {
if request.Trigger == "scheduled" || request.Trigger == "recovery" {
record.LastEnqueuedAt = now
}
for _, existing := range record.Runs {
- if existing.ID == request.RunID || coalescesPendingInternal(request, existing) {
+ // Manual and remote work never joins another run; other triggers join runs still waiting to start.
+ coalesces := request.ObservedAgentRun == nil && !separateTrigger(request.Trigger) && !separateTrigger(existing.Trigger) &&
+ (existing.Status == st.Queued || existing.Status == st.Waiting || existing.Status == st.Retrying)
+ if existing.ID == request.RunID || coalesces {
result = existing
return nil
}
@@ -121,34 +127,25 @@ func (q *Coordinator) Submit(ctx context.Context, request st.Request) (st.Run, e
result = existing
return nil
}
- pruneRunsInternal(record, now)
+ pruneRuns(record, now)
record.Runs = append(record.Runs, result)
return nil
})
if err == nil {
- q.observeRunInternal(ctx, result)
- q.signalInternal()
+ q.observeRun(ctx, result)
+ q.signal()
}
return result, err
}
-func (q *Coordinator) signalInternal() {
+func (q *Coordinator) signal() {
select {
case q.wake <- struct{}{}:
default:
}
}
-// Manual and remote work never joins another run. Scheduled and recovery work
-// joins an existing run only while it is still waiting to start.
-func coalescesPendingInternal(request st.Request, existing st.Run) bool {
- if request.ObservedAgentRun != nil || separateTriggerInternal(request.Trigger) || separateTriggerInternal(existing.Trigger) {
- return false
- }
- return existing.Status == st.Queued || existing.Status == st.Waiting || existing.Status == st.Retrying
-}
-
-func separateTriggerInternal(trigger string) bool {
+func separateTrigger(trigger string) bool {
switch trigger {
case "manual", "remote":
return true
@@ -164,22 +161,23 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, ne
first := !q.checkpointed[jobID]
recovered := false
var recoveredRun st.Run
- err := q.mutateInternal(ctx, "0", jobID, func(record *st.QueueRecord) error {
+ err := q.mutate(ctx, "0", jobID, func(record *st.QueueRecord) error {
recovered = false
recoveredRun = st.Run{}
now := time.Now().UTC()
if first && record.Schedule == schedule && !record.NextRun.IsZero() && !record.NextRun.After(now) && record.LastEnqueuedAt.Before(record.NextRun) {
- pending := false
- for _, run := range record.Runs {
- if !run.Status.Terminal() && run.Status != st.NeedsAttention {
- pending = true
- break
- }
- }
- if !pending {
+ if !slices.ContainsFunc(record.Runs, func(run st.Run) bool { return !run.Status.Terminal() && run.Status != st.NeedsAttention }) {
recovered = true
record.LastEnqueuedAt = now
- recoveredRun = st.Run{ID: uuid.New().String(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now}
+ recoveredRun = st.Run{
+ ID: uuid.New().String(),
+ JobID: jobID,
+ EnvironmentID: "0",
+ Trigger: "recovery",
+ Status: st.Queued,
+ CreatedAt: now,
+ UpdatedAt: now,
+ }
record.Runs = append(record.Runs, recoveredRun)
}
}
@@ -194,8 +192,8 @@ func (q *Coordinator) Checkpoint(ctx context.Context, jobID, schedule string, ne
if err == nil {
q.checkpointed[jobID] = true
if recovered {
- q.observeRunInternal(ctx, recoveredRun)
- q.signalInternal()
+ q.observeRun(ctx, recoveredRun)
+ q.signal()
}
}
return err
@@ -218,7 +216,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {
if q.execute == nil {
return errors.New("job executor unavailable")
}
- if err := q.importLegacyInternal(ctx); err != nil {
+ if err := q.importLegacy(ctx); err != nil {
return fmt.Errorf("import legacy job history: %w", err)
}
// Fail startup if persisted state cannot be read. Never replace it with empty state.
@@ -229,7 +227,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {
for _, record := range records {
changed := false
for _, run := range record.Runs {
- if normalizeLegacyRunInternal(&run) {
+ if normalizeLegacyRun(&run) {
changed = true
break
}
@@ -237,16 +235,16 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {
if !changed {
continue
}
- if mutateErr := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
+ if mutateErr := q.mutate(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
for i := range current.Runs {
- normalizeLegacyRunInternal(&current.Runs[i])
+ normalizeLegacyRun(&current.Runs[i])
}
return nil
}); mutateErr != nil {
return fmt.Errorf("normalize startup job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)
}
}
- if pruneErr := q.pruneInternal(ctx); pruneErr != nil {
+ if pruneErr := q.prune(ctx); pruneErr != nil {
return fmt.Errorf("prune startup job history: %w", pruneErr)
}
if errErr := ctx.Err(); errErr != nil {
@@ -260,7 +258,7 @@ func (q *Coordinator) Start(ctx, appCtx context.Context) error {
q.cancel = cancel
q.lifecycleCtx = runCtx
q.started = true
- go q.dispatchInternal(runCtx)
+ go q.dispatch(runCtx)
return nil
}
@@ -286,7 +284,7 @@ func (q *Coordinator) Stop(ctx context.Context) error {
}
}
-func (q *Coordinator) dispatchInternal(ctx context.Context) {
+func (q *Coordinator) dispatch(ctx context.Context) {
defer close(q.done)
initialRetention := true
@@ -294,20 +292,20 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {
fullRepair := true
nextRetention := time.Now()
for ctx.Err() == nil {
- q.retryActivitySyncInternal(ctx)
+ q.retryActivitySync(ctx)
if initialRetention || !time.Now().Before(nextRetention) {
- if err := q.pruneInternal(ctx); err != nil {
+ if err := q.prune(ctx); err != nil {
if ctx.Err() == nil {
slog.ErrorContext(ctx, "Job retention failed", "error", err)
}
} else {
- initialRetention = q.hasPendingActivitySyncInternal(ctx)
+ initialRetention = q.hasPendingActivitySync(ctx)
}
nextRetention = time.Now().Add(time.Hour)
}
nextWake := time.Now().Add(15 * time.Second)
enabled := q.enabled.Load()
- if err := q.repairInternal(ctx, fullRepair); err != nil {
+ if err := q.repair(ctx, fullRepair); err != nil {
if ctx.Err() == nil {
slog.ErrorContext(ctx, "Job queue dispatch failed", "error", err)
}
@@ -318,7 +316,7 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {
if nextRetention.Before(nextWake) {
nextWake = nextRetention
}
- if q.hasPendingActivitySyncInternal(ctx) {
+ if q.hasPendingActivitySync(ctx) {
activityRetry := time.Now().Add(5 * time.Second)
if activityRetry.Before(nextWake) {
nextWake = activityRetry
@@ -336,9 +334,10 @@ func (q *Coordinator) dispatchInternal(ctx context.Context) {
}
}
-func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) {
- // Keep the execution slot until the terminal result has been durably recorded.
- for {
+func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st.Outcome) error {
+ // Keep the execution slot while retrying, but release it after a bounded backoff.
+ wait := time.Second
+ for attempt := 1; ; attempt++ {
var conflictOwner string
var conflictStatus st.RunStatus
err := q.UpdateRun(ctx, run, func(current *st.Run) error {
@@ -349,7 +348,15 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st
now := time.Now().UTC()
current.Status = outcome.Status
current.UpdatedAt = now
- outcome = mergeTargetProgressInternal(current.Outcome, outcome)
+ for _, target := range current.Outcome.Targets {
+ if !slices.ContainsFunc(outcome.Targets, func(updated st.TargetOutcome) bool { return updated.ID == target.ID }) {
+ outcome.Targets = append(outcome.Targets, target)
+ }
+ }
+ outcome.ActivityID = cmp.Or(outcome.ActivityID, current.Outcome.ActivityID)
+ if len(outcome.Steps) == 0 {
+ outcome.Steps = current.Outcome.Steps
+ }
current.Outcome = outcome
if outcome.Status == st.Retrying || outcome.Status == st.Waiting {
delay := 5 * time.Second * time.Duration(1<<min(run.AttemptCount-1, 6))
@@ -384,36 +391,27 @@ func (q *Coordinator) persistOutcome(ctx context.Context, run st.Run, outcome st
"currentOwner", conflictOwner,
"currentStatus", conflictStatus,
"outcomeStatus", outcome.Status)
- return
+ return nil
}
if err == nil {
- return
+ return nil
}
- slog.ErrorContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "error", err)
- timer := time.NewTimer(time.Second)
+ if attempt >= 10 {
+ slog.ErrorContext(ctx, "Giving up persisting job result; releasing execution guard", "runId", run.ID, "attempts", attempt, "error", err)
+ return fmt.Errorf("persist job result: %w", err)
+ }
+ slog.WarnContext(ctx, "Unable to persist job result; retaining execution guard", "runId", run.ID, "attempt", attempt, "retryIn", wait, "error", err)
+ timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
- return
+ return fmt.Errorf("persist job result: %w", err)
case <-timer.C:
}
+ wait = min(wait*2, 30*time.Second)
}
}
-func (q *Coordinator) invokeInternal(ctx context.Context, run st.Run, reconcile bool) (outcome st.Outcome, err error) {
- defer func() {
- if recovered := recover(); recovered != nil {
- outcome = st.Outcome{Status: st.Failed, Message: "Job panicked; inspect logs before retrying"}
- err = fmt.Errorf("job panic: %v", recovered)
- slog.ErrorContext(ctx, "Job panicked", "runId", run.ID, "error", err)
- }
- }()
- if reconcile && q.reconcile != nil {
- return q.reconcile(ctx, run)
- }
- return q.execute(ctx, run)
-}
-
// Cancel marks an unstarted queued run canceled. Runs already attempted locally
// or dispatched remotely must be reconciled rather than canceled as pending work.
func (q *Coordinator) Cancel(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
@@ -435,11 +433,12 @@ func (q *Coordinator) Cancel(ctx context.Context, environmentID, jobID, runID st
if err == nil {
var state st.CoordinatorState
id := kit.SHA256Hex(environmentID + "\x00" + jobID)
- actorType := executorTypeInternal
- if strings.HasPrefix(jobID, "environment-health") || jobID == "docker-client-refresh" || jobID == "activity-sweep" {
- actorType = healthExecutorTypeInternal
- }
- if readErr := q.service.GetState(ctx, coordinatorTypeInternal, id, &state); readErr == nil {
+ actorType := kit.Ternary(
+ strings.HasPrefix(jobID, "environment-health") || jobID == "docker-client-refresh" || jobID == "activity-sweep",
+ healthExecutorType,
+ executorType,
+ )
+ if readErr := q.service.GetState(ctx, coordinatorType, id, &state); readErr == nil {
for _, intent := range state.Dispatches {
if intent.RunID == runID && intent.JobID != "" {
_ = q.service.DeleteJob(ctx, actorType, id, intent.JobID, actor.WithLiveJobsOnly())
@@ -473,12 +472,12 @@ func (q *Coordinator) Retry(ctx context.Context, environmentID, jobID, runID str
return nil
})
if err == nil {
- q.signalInternal()
+ q.signal()
}
return run, err
}
-func (q *Coordinator) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {
+func (q *Coordinator) nextRun(runs []st.Run) (*st.Run, time.Time) {
var interrupted *st.Run
blocked := false
for index := range runs {
@@ -510,33 +509,13 @@ func (q *Coordinator) nextRunInternal(runs []st.Run) (*st.Run, time.Time) {
return nil, nextRetry
}
-func mergeTargetProgressInternal(previous, outcome st.Outcome) st.Outcome {
- for _, target := range previous.Targets {
- found := false
- for _, updated := range outcome.Targets {
- if updated.ID == target.ID {
- found = true
- break
- }
- }
- if !found {
- outcome.Targets = append(outcome.Targets, target)
- }
- }
- outcome.ActivityID = cmp.Or(outcome.ActivityID, previous.ActivityID)
- if len(outcome.Steps) == 0 {
- outcome.Steps = previous.Steps
- }
- return outcome
-}
-
// ExecutionContext binds durable progress for work executing a run. An empty
// instanceID fences writes by run owner; otherwise by the active workflow instance.
func (q *Coordinator) ExecutionContext(ctx context.Context, run st.Run, instanceID string) context.Context {
ctx = utils.WithActivityBatchID(ctx, run.ID)
return jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {
return q.UpdateRun(ctx, run, func(current *st.Run) error {
- if !fencedInternal(current, run, instanceID) {
+ if !kit.Ternary(instanceID == "", current.Status == st.Running && current.Owner == run.Owner, ActiveWorkflow(*current, instanceID)) {
return ErrRunConflict
}
current.Outcome.Targets = jobcontext.Merge(current.Outcome.Targets, target)
@@ -545,10 +524,10 @@ func (q *Coordinator) ExecutionContext(ctx context.Context, run st.Run, instance
})
}
-// RecordSteps projects workflow step progress onto a running run.
+// RecordSteps projects workflow step progress onto a running run, fenced like ExecutionContext.
func (q *Coordinator) RecordSteps(ctx context.Context, run st.Run, instanceID string, steps []st.StepOutcome) error {
return q.UpdateRun(ctx, run, func(current *st.Run) error {
- if !fencedInternal(current, run, instanceID) {
+ if !kit.Ternary(instanceID == "", current.Status == st.Running && current.Owner == run.Owner, ActiveWorkflow(*current, instanceID)) {
return ErrRunConflict
}
current.Outcome.Steps = steps
@@ -556,14 +535,6 @@ func (q *Coordinator) RecordSteps(ctx context.Context, run st.Run, instanceID st
})
}
-// fencedInternal reports whether work may still write to a run.
-func fencedInternal(current *st.Run, run st.Run, instanceID string) bool {
- if instanceID == "" {
- return current.Status == st.Running && current.Owner == run.Owner
- }
- return ActiveWorkflow(*current, instanceID)
-}
-
// ActiveWorkflow reports whether a running run still follows a workflow instance.
func ActiveWorkflow(run st.Run, instanceID string) bool {
return run.Status == st.Running && slices.ContainsFunc(run.Outcome.Targets, func(target st.TargetOutcome) bool {
@@ -602,7 +573,7 @@ func (q *Coordinator) Resolve(ctx context.Context, environmentID, jobID, runID,
return nil
})
if err == nil {
- q.signalInternal()
+ q.signal()
}
return run, err
}
diff --git a/backend/pkg/scheduler/runs/coordinator_store.go b/backend/pkg/scheduler/runs/coordinator_store.go
index 18179dd47..6cbe0b069 100644
--- a/backend/pkg/scheduler/runs/coordinator_store.go
+++ b/backend/pkg/scheduler/runs/coordinator_store.go
@@ -19,21 +19,21 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/kv"
)
-const queuePrefixInternal = "jobs."
+const queuePrefix = "jobs."
-func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
+func (q *Coordinator) mutate(ctx context.Context, environmentID, jobID string, change func(*st.QueueRecord) error) error {
id := kit.SHA256Hex(environmentID + "\x00" + jobID)
q.runtime.NameActor(id, jobID+"@"+environmentID)
for range 20 {
var state st.CoordinatorState
- err := q.service.GetState(ctx, coordinatorTypeInternal, id, &state)
+ err := q.service.GetState(ctx, coordinatorType, id, &state)
if err != nil && !errors.Is(err, actor.ErrStateNotFound) {
return err
}
- if prepareMutationErr := q.prepareMutationInternal(&state, environmentID, jobID, change); prepareMutationErr != nil {
+ if prepareMutationErr := q.prepareMutation(&state, environmentID, jobID, change); prepareMutationErr != nil {
return prepareMutationErr
}
- result, err := q.service.Invoke(ctx, coordinatorTypeInternal, id, "replace", state)
+ result, err := q.service.Invoke(ctx, coordinatorType, id, "replace", state)
if err != nil {
return err
}
@@ -48,10 +48,10 @@ func (q *Coordinator) mutateInternal(ctx context.Context, environmentID, jobID s
return errors.New("job coordinator state is busy; retry mutation")
}
-func (q *Coordinator) prepareMutationInternal(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {
- normalizeRecordTimesInternal(&state.Record)
+func (q *Coordinator) prepareMutation(state *st.CoordinatorState, environmentID, jobID string, change func(*st.QueueRecord) error) error {
+ normalizeRecordTimes(&state.Record)
previousRuns := slices.Clone(state.Record.Runs)
- previous, versions, err := snapshotRunsInternal(state.Record)
+ previous, versions, err := snapshotRuns(state.Record)
if err != nil {
return err
}
@@ -61,28 +61,28 @@ func (q *Coordinator) prepareMutationInternal(state *st.CoordinatorState, enviro
if changeErr := change(&state.Record); changeErr != nil {
return changeErr
}
- preserveUnsyncedRunsInternal(state, previousRuns)
+ preserveUnsyncedRuns(state, previousRuns)
for i := range state.Record.Runs {
- q.associateActivityInternal(&state.Record.Runs[i])
- if markChangedActivityErr := markChangedActivityInternal(state, &state.Record.Runs[i], previous, versions); markChangedActivityErr != nil {
+ q.associateActivity(&state.Record.Runs[i])
+ if markChangedActivityErr := markChangedActivity(state, &state.Record.Runs[i], previous, versions); markChangedActivityErr != nil {
return markChangedActivityErr
}
}
for id, run := range state.Record.Receipts {
- q.associateActivityInternal(&run)
- if markChangedActivityErr2 := markChangedActivityInternal(state, &run, previous, versions); markChangedActivityErr2 != nil {
+ q.associateActivity(&run)
+ if markChangedActivityErr2 := markChangedActivity(state, &run, previous, versions); markChangedActivityErr2 != nil {
return markChangedActivityErr2
}
state.Record.Receipts[id] = run
}
- pruneDispatchesInternal(state)
+ pruneDispatches(state)
return nil
}
-func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]time.Time, error) {
+func snapshotRuns(record st.QueueRecord) (map[string][]byte, map[string]time.Time, error) {
encoded := make(map[string][]byte, len(record.Runs)+len(record.Receipts))
versions := make(map[string]time.Time, len(record.Runs)+len(record.Receipts))
- for _, run := range append(slices.Clone(record.Runs), receiptRunsInternal(record.Receipts)...) {
+ for _, run := range append(slices.Clone(record.Runs), receiptRuns(record.Receipts)...) {
data, err := json.Marshal(run)
if err != nil {
return nil, nil, err
@@ -93,7 +93,7 @@ func snapshotRunsInternal(record st.QueueRecord) (map[string][]byte, map[string]
return encoded, versions, nil
}
-func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run) {
+func preserveUnsyncedRuns(state *st.CoordinatorState, previous []st.Run) {
retained := make(map[string]st.Run, len(state.Record.Runs))
for _, run := range state.Record.Runs {
retained[run.ID] = run
@@ -133,7 +133,7 @@ func preserveUnsyncedRunsInternal(state *st.CoordinatorState, previous []st.Run)
state.Record.Runs = runs
}
-func pruneDispatchesInternal(state *st.CoordinatorState) {
+func pruneDispatches(state *st.CoordinatorState) {
attempts := make(map[string]int, len(state.Record.Runs))
for _, run := range state.Record.Runs {
attempts[run.ID] = run.AttemptCount
@@ -150,7 +150,7 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
records := []st.QueueRecord{}
cursor := ""
for {
- page, err := q.service.ListStates(ctx, coordinatorTypeInternal, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
+ page, err := q.service.ListStates(ctx, coordinatorType, &actor.ListStatesOpts{IncludeData: true, After: cursor, Limit: 100})
if err != nil {
return nil, fmt.Errorf("list job coordinator states: %w", err)
}
@@ -159,7 +159,7 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
if decodeErr := entry.Data.Decode(&state); decodeErr != nil {
return nil, fmt.Errorf("decode job coordinator %q: %w", entry.ActorID, decodeErr)
}
- normalizeRecordTimesInternal(&state.Record)
+ normalizeRecordTimes(&state.Record)
q.runtime.NameActor(entry.ActorID, state.Record.JobID+"@"+state.Record.EnvironmentID)
records = append(records, state.Record)
}
@@ -174,23 +174,23 @@ func (q *Coordinator) Records(ctx context.Context) ([]st.QueueRecord, error) {
// idempotency receipts. It returns ErrRunNotFound when no matching record exists.
func (q *Coordinator) Get(ctx context.Context, environmentID, jobID, runID string) (st.Run, error) {
var state st.CoordinatorState
- err := q.service.GetState(ctx, coordinatorTypeInternal, kit.SHA256Hex(environmentID+"\x00"+jobID), &state)
+ err := q.service.GetState(ctx, coordinatorType, kit.SHA256Hex(environmentID+"\x00"+jobID), &state)
if errors.Is(err, actor.ErrStateNotFound) {
return st.Run{}, ErrRunNotFound
}
if err != nil {
return st.Run{}, err
}
- normalizeRecordTimesInternal(&state.Record)
+ normalizeRecordTimes(&state.Record)
record := state.Record
for _, run := range record.Runs {
if run.ID == runID {
- q.associateActivityInternal(&run)
+ q.associateActivity(&run)
return run, nil
}
}
if run, ok := record.Receipts[runID]; ok {
- q.associateActivityInternal(&run)
+ q.associateActivity(&run)
return run, nil
}
return st.Run{}, ErrRunNotFound
@@ -224,7 +224,7 @@ func (q *Coordinator) List(ctx context.Context, environmentID, jobID string, pag
// and swap. The callback may run more than once and must not perform external side effects.
func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run) error) error {
wake := false
- err := q.mutateInternal(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {
+ err := q.mutate(ctx, run.EnvironmentID, run.JobID, func(record *st.QueueRecord) error {
for index := range record.Runs {
if record.Runs[index].ID == run.ID {
previous := record.Runs[index].Status
@@ -248,26 +248,26 @@ func (q *Coordinator) UpdateRun(ctx context.Context, run st.Run, change func(*st
return ErrRunNotFound
})
if err == nil && wake {
- q.signalInternal()
+ q.signal()
}
if err == nil {
- q.observeRunInternal(ctx, run)
+ q.observeRun(ctx, run)
}
return err
}
-func (q *Coordinator) pruneInternal(ctx context.Context) error {
+func (q *Coordinator) prune(ctx context.Context) error {
records, err := q.Records(ctx)
if err != nil {
return err
}
now := time.Now().UTC()
for _, record := range records {
- if !pruneRunsInternal(&record, now) {
+ if !pruneRuns(&record, now) {
continue
}
- if mutateErr := q.mutateInternal(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
- pruneRunsInternal(current, now)
+ if mutateErr := q.mutate(ctx, record.EnvironmentID, record.JobID, func(current *st.QueueRecord) error {
+ pruneRuns(current, now)
return nil
}); mutateErr != nil {
return fmt.Errorf("prune job %q in environment %q: %w", record.JobID, record.EnvironmentID, mutateErr)
@@ -276,7 +276,7 @@ func (q *Coordinator) pruneInternal(ctx context.Context) error {
return nil
}
-func pruneRunsInternal(record *st.QueueRecord, now time.Time) bool {
+func pruneRuns(record *st.QueueRecord, now time.Time) bool {
changed := false
terminal := 0
for index, run := range slices.Backward(record.Runs) {
@@ -287,7 +287,7 @@ func pruneRunsInternal(record *st.QueueRecord, now time.Time) bool {
if terminal <= 100 && now.Sub(run.UpdatedAt) < 7*24*time.Hour {
continue
}
- if separateTriggerInternal(run.Trigger) {
+ if separateTrigger(run.Trigger) {
if record.Receipts == nil {
record.Receipts = make(map[string]st.Run)
}
@@ -321,14 +321,14 @@ func (q *Coordinator) SetExecutor(execute, reconcile func(context.Context, st.Ru
func (q *Coordinator) ScheduleState(ctx context.Context, jobID string) (st.QueueRecord, error) {
var state st.CoordinatorState
- err := q.service.GetState(ctx, coordinatorTypeInternal, kit.SHA256Hex("0\x00"+jobID), &state)
- normalizeRecordTimesInternal(&state.Record)
+ err := q.service.GetState(ctx, coordinatorType, kit.SHA256Hex("0\x00"+jobID), &state)
+ normalizeRecordTimes(&state.Record)
return state.Record, err
}
-func (q *Coordinator) Activate() { q.enabled.Store(true); q.signalInternal() }
+func (q *Coordinator) Activate() { q.enabled.Store(true); q.signal() }
-func receiptRunsInternal(receipts map[string]st.Run) []st.Run {
+func receiptRuns(receipts map[string]st.Run) []st.Run {
runs := make([]st.Run, 0, len(receipts))
for _, run := range receipts {
runs = append(runs, run)
@@ -336,8 +336,8 @@ func receiptRunsInternal(receipts map[string]st.Run) []st.Run {
return runs
}
-func markChangedActivityInternal(state *st.CoordinatorState, run *st.Run, previous map[string][]byte, previousUpdated map[string]time.Time) error {
- normalizeRunTimesInternal(run)
+func markChangedActivity(state *st.CoordinatorState, run *st.Run, previous map[string][]byte, previousUpdated map[string]time.Time) error {
+ normalizeRunTimes(run)
if run.ActivityID == "" {
return nil
}
@@ -358,7 +358,7 @@ func markChangedActivityInternal(state *st.CoordinatorState, run *st.Run, previo
return nil
}
-func markActivitySyncInternal(state *st.CoordinatorState, run st.Run) {
+func markActivitySync(state *st.CoordinatorState, run st.Run) {
if run.ActivityID == "" {
return
}
@@ -368,35 +368,35 @@ func markActivitySyncInternal(state *st.CoordinatorState, run st.Run) {
state.PendingActivitySync[run.ID] = run.UpdatedAt
}
-func normalizeRecordTimesInternal(record *st.QueueRecord) {
+func normalizeRecordTimes(record *st.QueueRecord) {
record.LastEnqueuedAt = record.LastEnqueuedAt.UTC()
record.NextRun = record.NextRun.UTC()
for i := range record.Runs {
- normalizeRunTimesInternal(&record.Runs[i])
+ normalizeRunTimes(&record.Runs[i])
}
for id, run := range record.Receipts {
- normalizeRunTimesInternal(&run)
+ normalizeRunTimes(&run)
record.Receipts[id] = run
}
}
-func normalizeRunTimesInternal(run *st.Run) {
+func normalizeRunTimes(run *st.Run) {
run.CreatedAt = run.CreatedAt.UTC()
run.UpdatedAt = run.UpdatedAt.UTC()
- normalizeTimePointerInternal(run.StartedAt)
- normalizeTimePointerInternal(run.FinishedAt)
- normalizeTimePointerInternal(run.NextAttempt)
- normalizeTimePointerInternal(run.LastConfirmedAt)
+ normalizeTimePointer(run.StartedAt)
+ normalizeTimePointer(run.FinishedAt)
+ normalizeTimePointer(run.NextAttempt)
+ normalizeTimePointer(run.LastConfirmedAt)
if run.Resolution != nil {
run.Resolution.ResolvedAt = run.Resolution.ResolvedAt.UTC()
}
for i := range run.Attempts {
run.Attempts[i].StartedAt = run.Attempts[i].StartedAt.UTC()
- normalizeTimePointerInternal(run.Attempts[i].FinishedAt)
+ normalizeTimePointer(run.Attempts[i].FinishedAt)
}
}
-func normalizeLegacyRunInternal(run *st.Run) bool {
+func normalizeLegacyRun(run *st.Run) bool {
automaticResolution := run.Status == st.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == user.SystemUser.Username
if run.Status != st.NeedsAttention && !automaticResolution {
return false
@@ -417,7 +417,7 @@ func normalizeLegacyRunInternal(run *st.Run) bool {
return true
}
-func normalizeTimePointerInternal(value *time.Time) {
+func normalizeTimePointer(value *time.Time) {
if value != nil {
*value = value.UTC()
}
diff --git a/backend/pkg/scheduler/runs/resolution_test.go b/backend/pkg/scheduler/runs/resolution_test.go
index a8d5fc078..5b7f3b062 100644
--- a/backend/pkg/scheduler/runs/resolution_test.go
+++ b/backend/pkg/scheduler/runs/resolution_test.go
@@ -15,7 +15,7 @@ import (
francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing"
)
-func newResolutionQueueInternal(t *testing.T) *Coordinator {
+func newResolutionQueue(t *testing.T) *Coordinator {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
@@ -32,7 +32,7 @@ func newResolutionQueueInternal(t *testing.T) *Coordinator {
}
func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
ctx := t.Context()
run, err := q.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "manual"})
require.NoError(t, err)
@@ -48,7 +48,7 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
require.NoError(t, err)
history, err := q.List(ctx, "0", "auto-update", 1, 20)
require.NoError(t, err)
- selected, _ := q.nextRunInternal(history.Runs)
+ selected, _ := q.nextRun(history.Runs)
require.NotNil(t, selected)
require.Equal(t, pending.ID, selected.ID)
for len(q.wake) > 0 {
@@ -69,7 +69,7 @@ func TestResolvePreservesEvidenceWithoutBlockingSchedule(t *testing.T) {
}
history, err = q.List(ctx, "0", "auto-update", 1, 20)
require.NoError(t, err)
- selected, _ = q.nextRunInternal(history.Runs)
+ selected, _ = q.nextRun(history.Runs)
require.NotNil(t, selected)
require.Equal(t, pending.ID, selected.ID)
duplicate, err := q.Resolve(ctx, "0", "auto-update", run.ID, "different operator")
@@ -139,7 +139,7 @@ func TestResolveRejectsUnsafeStates(t *testing.T) {
{name: "uncertain remote", status: st.NeedsAttention, environment: "remote", delivered: true},
} {
t.Run(test.name, func(t *testing.T) {
- q := newResolutionQueueInternal(t)
+ q := newResolutionQueue(t)
run, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: test.environment})
require.NoError(t, err)
require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error {
diff --git a/backend/pkg/utils/francis/provider.go b/backend/pkg/utils/francis/provider.go
index aaad6b958..8c6c49243 100644
--- a/backend/pkg/utils/francis/provider.go
+++ b/backend/pkg/utils/francis/provider.go
@@ -66,7 +66,7 @@ func StoreURL(databaseURL string) (string, error) {
// The sibling file must be creatable and writable whatever mode opens arcane.db.
query.Del("mode")
if !sqliteutil.HasPragma(query, "busy_timeout") {
- query.Add("_pragma", "busy_timeout(2500)")
+ query.Add("_pragma", "busy_timeout(10000)")
}
// NORMAL stops WAL commits from holding the write lock through an fsync.
if !sqliteutil.HasPragma(query, "synchronous") {
diff --git a/backend/pkg/utils/francis/provider_test.go b/backend/pkg/utils/francis/provider_test.go
index 1b2a65b7e..dd5ecef1a 100644
--- a/backend/pkg/utils/francis/provider_test.go
+++ b/backend/pkg/utils/francis/provider_test.go
@@ -18,7 +18,7 @@ func TestMigrateLegacyStore(t *testing.T) {
databaseURL := "file:" + mainPath + "?_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)&_txlock=immediate"
storeURL, err := StoreURL(databaseURL)
require.NoError(t, err)
- const defaults = "_pragma=busy_timeout%282500%29&_pragma=synchronous%28NORMAL%29"
+ const defaults = "_pragma=busy_timeout%2810000%29&_pragma=synchronous%28NORMAL%29"
require.Equal(t, "file:"+filepath.Join(filepath.Dir(mainPath), "arcane.francis.db")+"?_pragma=journal_mode%28WAL%29&"+defaults+"&_txlock=immediate", storeURL)
for input, expected := range map[string]string{
"file:/data/arcane%23prod.db?_txlock=immediate": "file:/data/arcane%23prod.francis.db?" + defaults + "&_txlock=immediate",
@@ -26,8 +26,8 @@ func TestMigrateLegacyStore(t *testing.T) {
"file:data/arcane%23prod%2Edb": "file:data/arcane%23prod.francis.db?" + defaults,
"file:data/arcane": "file:data/arcane.francis.db?" + defaults,
"file:arcane.db?_pragma=busy_timeout(15000)&_pragma=foreign_keys(1)&_fk=1": "file:arcane.francis.db?_pragma=busy_timeout%2815000%29&_pragma=synchronous%28NORMAL%29",
- "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%282500%29",
- "file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%282500%29",
+ "file:arcane.db?_sync=FULL&mode=rw": "file:arcane.francis.db?_pragma=synchronous%28FULL%29&_pragma=busy_timeout%2810000%29",
+ "file:arcane.db?_pragma=journal_mode(DELETE)&_pragma=synchronous(EXTRA)": "file:arcane.francis.db?_pragma=journal_mode%28DELETE%29&_pragma=synchronous%28EXTRA%29&_pragma=busy_timeout%2810000%29",
"postgres://arcane@localhost/arcane?sslmode=disable": "postgres://arcane@localhost/arcane?sslmode=disable",
} {
storeURL, err = StoreURL(input)
diff --git a/backend/pkg/utils/imageref/repodigest.go b/backend/pkg/utils/imageref/repodigest.go
new file mode 100644
index 000000000..ef589e3af
--- /dev/null
+++ b/backend/pkg/utils/imageref/repodigest.go
@@ -0,0 +1,14 @@
+package imageref
+
+import "strings"
+
+// RepoDigestRepository returns the repository of the first usable
+// "repo@digest" entry in an image's RepoDigests.
+func RepoDigestRepository(repoDigests []string) (string, bool) {
+ for _, repoDigest := range repoDigests {
+ if repository, _, found := strings.CutLast(repoDigest, "@"); repoDigest != "<none>@<none>" && found && repository != "" {
+ return repository, true
+ }
+ }
+ return "", false
+}
diff --git a/backend/pkg/utils/imageref/repodigest_test.go b/backend/pkg/utils/imageref/repodigest_test.go
new file mode 100644
index 000000000..1c4a53001
--- /dev/null
+++ b/backend/pkg/utils/imageref/repodigest_test.go
@@ -0,0 +1,38 @@
+package imageref
+
+import "testing"
+
+func TestRepoDigestRepository(t *testing.T) {
+ tests := []struct {
+ name string
+ repoDigests []string
+ want string
+ wantOK bool
+ }{
+ {name: "nil"},
+ {name: "none placeholder only", repoDigests: []string{"<none>@<none>"}},
+ {name: "missing at sign", repoDigests: []string{"nginx"}},
+ {name: "empty repository", repoDigests: []string{"@sha256:abc"}},
+ {
+ name: "first usable entry wins",
+ repoDigests: []string{"<none>@<none>", "ghcr.io/getarcaneapp/arcane@sha256:abc", "nginx@sha256:def"},
+ want: "ghcr.io/getarcaneapp/arcane",
+ wantOK: true,
+ },
+ {
+ name: "registry with port",
+ repoDigests: []string{"localhost:5000/app@sha256:abc"},
+ want: "localhost:5000/app",
+ wantOK: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, ok := RepoDigestRepository(tt.repoDigests)
+ if got != tt.want || ok != tt.wantOK {
+ t.Fatalf("RepoDigestRepository(%q) = (%q, %v), want (%q, %v)", tt.repoDigests, got, ok, tt.want, tt.wantOK)
+ }
+ })
+ }
+}
diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yaml
index ab736a43d..3dcb749e5 100644
--- a/docker/compose.dev.yaml
+++ b/docker/compose.dev.yaml
@@ -60,11 +60,12 @@ services:
- PGID=1000
- ENCRYPTION_KEY=dev-encryption-key-replace-in-production-must-be-32-chars
- JWT_SECRET=dev-jwt-secret-replace-in-production-must-be-long-enough
- - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate
+ - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate
working_dir: /app/backend
networks:
- arcane-dev
restart: unless-stopped
+ stop_grace_period: 15s
healthcheck:
test: ['CMD-SHELL', 'curl -fsS http://localhost:3552/api/health >/dev/null || exit 1']
interval: 30s
@@ -92,7 +93,7 @@ services:
- EDGE_TRANSPORT=websocket
- ENCRYPTION_KEY=dev-encryption-key-replace-in-production-must-be-32-chars
- JWT_SECRET=dev-jwt-secret-replace-in-production-must-be-long-enough
- - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate
+ - DATABASE_URL=file:/app/data/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- agent-dev-data:/app/data
@@ -108,6 +109,7 @@ services:
networks:
- arcane-dev
restart: unless-stopped
+ stop_grace_period: 15s
healthcheck:
test: ['CMD-SHELL', 'curl -fsS http://localhost:3553/api/health >/dev/null || exit 1']
interval: 30s
diff --git a/docker/examples/compose.agent-edge.yaml b/docker/examples/compose.agent-edge.yaml
index 75aefba82..4bc94dd16 100644
--- a/docker/examples/compose.agent-edge.yaml
+++ b/docker/examples/compose.agent-edge.yaml
@@ -1,6 +1,6 @@
services:
arcane-edge-agent:
- image: ghcr.io/getarcaneapp/agent:latest
+ image: getarcaneapp/agent:latest
container_name: arcane-edge-agent
# No ports exposed - agent connects outbound to manager tunnel transport
environment:
@@ -20,6 +20,7 @@ services:
# This allows Arcane to detect its own container ID more reliably
cgroup: host
restart: unless-stopped
+ stop_grace_period: 15s
volumes:
agent-data:
diff --git a/docker/examples/compose.agent.yaml b/docker/examples/compose.agent.yaml
index 6dd4e6568..83c8caaec 100644
--- a/docker/examples/compose.agent.yaml
+++ b/docker/examples/compose.agent.yaml
@@ -1,6 +1,6 @@
services:
arcane-agent:
- image: ghcr.io/getarcaneapp/agent:latest
+ image: getarcaneapp/agent:latest
container_name: arcane-agent
ports:
- '3553:3553'
@@ -21,6 +21,7 @@ services:
# This allows Arcane to detect its own container ID more reliably
cgroup: host
restart: unless-stopped
+ stop_grace_period: 15s
volumes:
agent-data:
diff --git a/docker/examples/compose.basic.yaml b/docker/examples/compose.basic.yaml
index d864c8a12..76f04313a 100644
--- a/docker/examples/compose.basic.yaml
+++ b/docker/examples/compose.basic.yaml
@@ -1,6 +1,6 @@
services:
arcane:
- image: ghcr.io/getarcaneapp/manager:latest
+ image: getarcaneapp/manager:latest
container_name: arcane
ports:
- '3552:3552'
@@ -29,6 +29,7 @@ services:
retries: 5
start_period: 15s
restart: unless-stopped
+ stop_grace_period: 15s
volumes:
arcane-data:
diff --git a/docker/examples/compose.otel.yaml b/docker/examples/compose.otel.yaml
index 10f2e94ae..0ddf7f4ea 100644
--- a/docker/examples/compose.otel.yaml
+++ b/docker/examples/compose.otel.yaml
@@ -1,6 +1,6 @@
services:
arcane:
- image: ghcr.io/getarcaneapp/manager:latest
+ image: getarcaneapp/manager:latest
container_name: arcane
ports:
- '3552:3552'
@@ -20,6 +20,7 @@ services:
depends_on:
- otel-collector
restart: unless-stopped
+ stop_grace_period: 15s
otel-collector:
image: otel/opentelemetry-collector-contrib:latest
diff --git a/docker/examples/compose.proxy.yaml b/docker/examples/compose.proxy.yaml
index 14444c5c4..c64d3464c 100644
--- a/docker/examples/compose.proxy.yaml
+++ b/docker/examples/compose.proxy.yaml
@@ -38,7 +38,7 @@ services:
- no-new-privileges:true
arcane:
- image: ghcr.io/getarcaneapp/arcane:latest
+ image: getarcaneapp/arcane:latest
container_name: arcane
ports:
- '3552:3552'
@@ -65,6 +65,7 @@ services:
# This allows Arcane to detect its own container ID more reliably
cgroup: host
restart: unless-stopped
+ stop_grace_period: 15s
networks:
arcane-internal:
diff --git a/just/dev.just b/just/dev.just
index f92303cd2..7242353a8 100644
--- a/just/dev.just
+++ b/just/dev.just
@@ -28,7 +28,7 @@ agent:
manager_api_url="${MANAGER_API_URL:-https://localhost:3552}"
edge_mtls_assets_dir="${EDGE_MTLS_ASSETS_DIR:-./.tmp/edge-test-agent/edge-mtls-agent}"
edge_mtls_ca_file="${EDGE_MTLS_CA_FILE:-./backend/local-manager.crt}"
- database_url="${DATABASE_URL:-file:./.tmp/edge-test-agent/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(2500)&_txlock=immediate}"
+ database_url="${DATABASE_URL:-file:./.tmp/edge-test-agent/arcane.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(10000)&_txlock=immediate}"
projects_directory="${PROJECTS_DIRECTORY:-./.tmp/edge-test-agent/projects}"
git_work_dir="${GIT_WORK_DIR:-./.tmp/edge-test-agent/git}"
jwt_secret="${JWT_SECRET:-local-edge-test-jwt-secret-please-change}"
diff --git a/types/system/health.go b/types/system/health.go
index 07df4b0ec..a7bf4de9b 100644
--- a/types/system/health.go
+++ b/types/system/health.go
@@ -7,3 +7,15 @@ type HealthResponse struct {
// Required: true
Status string `json:"status"`
}
+
+// ReadinessResponse is the body of the readiness probe; Checks maps each dependency to "ok" or its error.
+type ReadinessResponse struct {
+ // Status is "ready" or "not_ready".
+ //
+ // Required: true
+ Status string `json:"status"`
+ // Checks maps each dependency name to "ok" or its error.
+ //
+ // Required: true
+ Checks map[string]string `json:"checks"`
+}