Skip to content

Modern Docker Management, Designed for Everyone

fix(backend): prevent stale cache fills and harden stream cleanup (#4267)

Kyle Mendellb48b11fdf738
diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.go
index f11d372d6..67804ad58 100644
--- a/backend/api/ws/container_stats.go
+++ b/backend/api/ws/container_stats.go
@@ -102,9 +102,7 @@ func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hub
if cleanupTimer != timer {
return
}
- if existing, ok := h.containerStatsHubs.Load(containerID); ok && existing == hub {
- h.containerStatsHubs.Delete(containerID)
- }
+ h.containerStatsHubs.CompareAndDelete(containerID, hub)
slog.Debug("container stats hub idle, cleaning up upstream stream", "containerID", containerID)
cleanupTimer = nil
cancel()
@@ -128,7 +126,9 @@ func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hub
defer close(statsChan)
err := h.containerService.StreamStats(ctx, containerID, statsChan)
+ h.containerStatsHubs.CompareAndDelete(containerID, hub)
if err == nil || errors.Is(err, context.Canceled) || ctx.Err() != nil {
+ cancel()
return
}
@@ -143,7 +143,6 @@ func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hub
}); marshalErr == nil {
hub.Broadcast(b)
}
- h.containerStatsHubs.CompareAndDelete(containerID, hub)
// Cancelling immediately would race hub.Run between ctx.Done and the
// queued error frame, dropping the very message clients need. Give the
diff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.go
index fba2911ae..2ba41e64a 100644
--- a/backend/api/ws/handler.go
+++ b/backend/api/ws/handler.go
@@ -32,18 +32,19 @@ var defaultWebSocketMetrics = wshub.NewWebSocketMetrics()
// WebSocketHandler consolidates all WebSocket and streaming endpoints.
// REST endpoints are handled by Huma handlers.
type WebSocketHandler struct {
- projectService *project.ProjectService
- containerService *container.ContainerService
- swarmService *swarm.SwarmService
- systemService *system.SystemService
- diagnosticsService *diagnostics.DiagnosticsService
- checkWSOrigin func(*http.Request) bool
- wsMetrics *wshub.WebSocketMetrics
- activeConnections sync.Map
- logStreamsMu sync.Mutex
- logStreams map[string]*wsLogStream
- cpuCache concurrency.Snapshot[float64]
- systemStaticInfo struct {
+ projectService *project.ProjectService
+ containerService *container.ContainerService
+ swarmService *swarm.SwarmService
+ systemService *system.SystemService
+ diagnosticsService *diagnostics.DiagnosticsService
+ checkWSOrigin func(*http.Request) bool
+ wsMetrics *wshub.WebSocketMetrics
+ activeConnectionsMu sync.Mutex
+ activeConnections map[string]int
+ logStreamsMu sync.Mutex
+ logStreams map[string]*wsLogStream
+ cpuCache concurrency.Snapshot[float64]
+ systemStaticInfo struct {
once sync.Once
cpuCount int
hostname string
diff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.go
index 69afaa0d9..a25f6df73 100644
--- a/backend/api/ws/system_stats.go
+++ b/backend/api/ws/system_stats.go
@@ -7,7 +7,6 @@ import (
"net/http"
"runtime"
"sync"
- "sync/atomic"
"time"
"github.com/coder/websocket"
@@ -37,30 +36,27 @@ const (
// ============================================================================
// checkRateLimitInternal checks and applies rate limiting for WebSocket connections.
-// Returns the counter and whether the connection should be allowed.
-func (h *WebSocketHandler) checkRateLimitInternal(clientIP string) (*int32, bool) {
- connCount, _ := h.activeConnections.LoadOrStore(clientIP, new(int32))
- count, ok := connCount.(*int32)
- if !ok {
- return nil, false
+func (h *WebSocketHandler) checkRateLimitInternal(clientIP string) bool {
+ h.activeConnectionsMu.Lock()
+ defer h.activeConnectionsMu.Unlock()
+ if h.activeConnections[clientIP] >= 5 {
+ return false
}
-
- currentCount := atomic.AddInt32(count, 1)
- if currentCount > 5 {
- atomic.AddInt32(count, -1)
- return nil, false
+ if h.activeConnections == nil {
+ h.activeConnections = make(map[string]int)
}
- return count, true
+ h.activeConnections[clientIP]++
+ return true
}
// releaseRateLimitInternal decrements the connection counter and cleans up if needed.
-func (h *WebSocketHandler) releaseRateLimitInternal(clientIP string, count *int32) {
- newCount := atomic.AddInt32(count, -1)
- if newCount <= 0 {
- // CompareAndDelete, not Delete: a plain delete removed whatever counter
- // was stored for this IP, which after a racing reconnect is a live one —
- // wiping its count and letting that IP exceed the limit.
- h.activeConnections.CompareAndDelete(clientIP, count)
+func (h *WebSocketHandler) releaseRateLimitInternal(clientIP string) {
+ h.activeConnectionsMu.Lock()
+ defer h.activeConnectionsMu.Unlock()
+ if count := h.activeConnections[clientIP]; count > 1 {
+ h.activeConnections[clientIP] = count - 1
+ } else {
+ delete(h.activeConnections, clientIP)
}
}
@@ -97,7 +93,7 @@ func (h *WebSocketHandler) acquireSystemStatsSamplerInternal(ctx context.Context
closeReady := sync.OnceFunc(func() {
close(ready)
})
- if !h.initializeCPUCacheCtx(samplerCtx) {
+ if !h.initializeCPUCacheInternal(samplerCtx) {
closeReady()
return
}
@@ -362,8 +358,8 @@ func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUSta
return gpuData, len(gpuData)
}
-// initializeCPUCacheCtx performs initial CPU sampling and returns early if the sampler is canceled.
-func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {
+// initializeCPUCacheInternal permits unavailable CPU samples but stops on cancellation.
+func (h *WebSocketHandler) initializeCPUCacheInternal(ctx context.Context) bool {
result := make(chan float64, 1)
go func() {
@@ -377,10 +373,12 @@ func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {
case <-ctx.Done():
return false
case val, ok := <-result:
- if !ok || ctx.Err() != nil {
+ if ctx.Err() != nil {
return false
}
- h.cpuCache.Store(val)
+ if ok {
+ h.cpuCache.Store(val)
+ }
return true
}
}
@@ -424,11 +422,10 @@ func (h *WebSocketHandler) getCachedCgroupLimitsInternal() *cgroup.Limits {
func (h *WebSocketHandler) SystemStats(c *echo.Context) error {
clientIP := c.RealIP()
- count, allowed := h.checkRateLimitInternal(clientIP)
- if !allowed {
+ if !h.checkRateLimitInternal(clientIP) {
return c.JSON(http.StatusTooManyRequests, map[string]any{"success": false, "error": "Too many concurrent stats connections from this IP"})
}
- defer h.releaseRateLimitInternal(clientIP, count)
+ defer h.releaseRateLimitInternal(clientIP)
conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindSystemStats, "")
if !ok {
diff --git a/backend/internal/auth/service.go b/backend/internal/auth/service.go
index 59b23a9ca..3216e4193 100644
--- a/backend/internal/auth/service.go
+++ b/backend/internal/auth/service.go
@@ -10,6 +10,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
"uuid"
@@ -75,7 +76,9 @@ type AuthService struct {
// skips signature verification and the user/session lookups; revocation in
// this process purges entries, revocation by another process is visible
// once the TTL lapses.
- tokenCache *hot.HotCache[string, verifiedTokenEntry]
+ tokenCache *hot.HotCache[string, verifiedTokenEntry]
+ cacheGen atomic.Uint64
+ cacheFillMu sync.Mutex
}
func NewAuthService(userService *user.UserService, settingsService *settings.SettingsService, eventService *event.EventService, sessionService *session.SessionService, roleService *role.RoleService, cfg *config.Config) *AuthService {
@@ -820,6 +823,7 @@ func (s *AuthService) cachedVerificationInternal(tokenHash string) (*common.User
}
func (s *AuthService) verifyTokenClaimsInternal(ctx context.Context, tokenHash string, claims *accessTokenClaims) (*common.User, string, error) {
+ gen := s.cacheGen.Load()
if claims.AppVersion != "" && claims.AppVersion != config.Version {
slog.InfoContext(ctx, "Token version mismatch detected", "tokenVersion", claims.AppVersion, "currentVersion", config.Version, "user", claims.Username)
return nil, "", common.ErrTokenVersionMismatch
@@ -847,7 +851,11 @@ func (s *AuthService) verifyTokenClaimsInternal(ctx context.Context, tokenHash s
return nil, "", err
}
- s.tokenCache.Set(tokenHash, verifiedTokenEntry{User: *dbUser, SessionID: userSession.ID, TokenExpiresAt: claims.ExpiresAt, SessionExpiresAt: userSession.ExpiresAt})
+ s.cacheFillMu.Lock()
+ if s.cacheGen.Load() == gen {
+ s.tokenCache.Set(tokenHash, verifiedTokenEntry{User: *dbUser, SessionID: userSession.ID, TokenExpiresAt: claims.ExpiresAt, SessionExpiresAt: userSession.ExpiresAt})
+ }
+ s.cacheFillMu.Unlock()
return dbUser, userSession.ID, nil
}
@@ -895,9 +903,16 @@ func (s *AuthService) InvalidateUserTokenCache(userID string) {
if s.tokenCache == nil || strings.TrimSpace(userID) == "" {
return
}
+ s.invalidateTokenCacheInternal(func(entry verifiedTokenEntry) bool { return entry.User.ID == userID })
+}
+
+func (s *AuthService) invalidateTokenCacheInternal(matches func(verifiedTokenEntry) bool) {
+ s.cacheFillMu.Lock()
+ defer s.cacheFillMu.Unlock()
+ s.cacheGen.Add(1)
keys := make([]string, 0)
s.tokenCache.Range(func(key string, entry verifiedTokenEntry) bool {
- if entry.User.ID == userID {
+ if matches(entry) {
keys = append(keys, key)
}
return true
@@ -909,15 +924,11 @@ func (s *AuthService) RevokeSession(ctx context.Context, sessionID string) error
if s.sessionService == nil {
return nil
}
- keys := make([]string, 0)
- s.tokenCache.Range(func(key string, entry verifiedTokenEntry) bool {
- if entry.SessionID == sessionID {
- keys = append(keys, key)
- }
- return true
- })
- s.tokenCache.DeleteMany(keys)
- return s.sessionService.RevokeSession(ctx, sessionID)
+ if err := s.sessionService.RevokeSession(ctx, sessionID); err != nil {
+ return err
+ }
+ s.invalidateTokenCacheInternal(func(entry verifiedTokenEntry) bool { return entry.SessionID == sessionID })
+ return nil
}
// LogoutAllOtherSessions revokes every active session for userID except
@@ -926,15 +937,13 @@ func (s *AuthService) LogoutAllOtherSessions(ctx context.Context, userID, curren
if s.sessionService == nil {
return nil
}
- keys := make([]string, 0)
- s.tokenCache.Range(func(key string, entry verifiedTokenEntry) bool {
- if entry.User.ID == userID && entry.SessionID != currentSessionID {
- keys = append(keys, key)
- }
- return true
+ if err := s.sessionService.RevokeAllUserSessionsExcept(ctx, userID, currentSessionID); err != nil {
+ return err
+ }
+ s.invalidateTokenCacheInternal(func(entry verifiedTokenEntry) bool {
+ return entry.User.ID == userID && entry.SessionID != currentSessionID
})
- s.tokenCache.DeleteMany(keys)
- return s.sessionService.RevokeAllUserSessionsExcept(ctx, userID, currentSessionID)
+ return nil
}
func (s *AuthService) createSessionAndTokensInternal(ctx context.Context, user *common.User, meta auth.SessionMeta) (*TokenPair, error) {
diff --git a/backend/internal/passkey/handler.go b/backend/internal/passkey/handler.go
index f5233b4d2..e6d5bec30 100644
--- a/backend/internal/passkey/handler.go
+++ b/backend/internal/passkey/handler.go
@@ -633,6 +633,7 @@ func (h *PasskeyHandler) DisableMFA(ctx context.Context, input *MFASettingsInput
if err := h.passkeyService.DisableMFA(ctx, userModel.ID, sessionID, input.StepUpToken); err != nil {
return nil, passkeyHTTPErrorInternal(err)
}
+ h.authService.InvalidateUserTokenCache(userModel.ID)
return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Passkey MFA disabled"}}}, nil
}
diff --git a/backend/internal/role/service.go b/backend/internal/role/service.go
index d16119863..e963ddfe5 100644
--- a/backend/internal/role/service.go
+++ b/backend/internal/role/service.go
@@ -9,6 +9,8 @@ import (
"maps"
"slices"
"strings"
+ "sync"
+ "sync/atomic"
"time"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
@@ -32,9 +34,12 @@ const (
)
type RoleService struct {
- db *database.DB
- userCache *hot.HotCache[string, *authz.PermissionSet]
- apiKeyCache *hot.HotCache[string, *authz.PermissionSet]
+ db *database.DB
+ userCache *hot.HotCache[string, *authz.PermissionSet]
+ apiKeyCache *hot.HotCache[string, *authz.PermissionSet]
+ cacheFillMu sync.Mutex
+ userCacheGen atomic.Uint64
+ apiKeyCacheGen atomic.Uint64
}
func NewRoleService(db *database.DB) *RoleService {
@@ -256,6 +261,7 @@ func (s *RoleService) UpdateRole(ctx context.Context, id, name string, descripti
return nil, err
}
var out Role
+ var affected []string
err := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {
var existing Role
if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", id).First(&existing).Error; err != nil {
@@ -276,9 +282,11 @@ func (s *RoleService) UpdateRole(ctx context.Context, id, name string, descripti
return common.Classify(common.ErrRoleNameTaken, errors.New("Role name already in use"))
}
}
- if _, err := lockAssignedUserRowsInternal(tx, id); err != nil {
+ ids, err := lockAssignedUserRowsInternal(tx, id)
+ if err != nil {
return err
}
+ affected = ids
existing.Name = input.Name
existing.Description = input.Description
existing.Permissions = permissions
@@ -291,7 +299,9 @@ func (s *RoleService) UpdateRole(ctx context.Context, id, name string, descripti
if err != nil {
return nil, err
}
- s.invalidateUsersAssignedToInternal(ctx, id)
+ for _, userID := range affected {
+ s.InvalidateUser(userID)
+ }
return &out, nil
}
@@ -327,7 +337,7 @@ func (s *RoleService) DeleteRole(ctx context.Context, id string) error {
// cache-miss cannot re-populate with stale data from the not-yet-visible
// delete. Consistent with UpdateRole / SetUserAssignments.
for _, uid := range affected {
- s.userCache.Delete(uid)
+ s.InvalidateUser(uid)
}
return nil
}
@@ -406,7 +416,7 @@ func (s *RoleService) replaceUserAssignmentsForSourceInternal(ctx context.Contex
if err != nil {
return err
}
- s.userCache.Delete(userID)
+ s.InvalidateUser(userID)
return nil
}
@@ -538,11 +548,16 @@ func (s *RoleService) ResolvePermissions(ctx context.Context, user *common.User)
if ps, ok, _ := s.userCache.Get(user.ID); ok {
return ps, nil
}
+ gen := s.userCacheGen.Load()
ps, err := s.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), user.ID)
if err != nil {
return nil, err
}
- s.userCache.Set(user.ID, ps)
+ s.cacheFillMu.Lock()
+ if s.userCacheGen.Load() == gen {
+ s.userCache.Set(user.ID, ps)
+ }
+ s.cacheFillMu.Unlock()
return ps, nil
}
@@ -653,11 +668,16 @@ func (s *RoleService) ResolveApiKeyPermissions(ctx context.Context, apiKeyID str
if ps, ok, _ := s.apiKeyCache.Get(apiKeyID); ok {
return ps, nil
}
+ gen := s.apiKeyCacheGen.Load()
permissions, err := s.resolveApiKeyPermissionsInDBInternal(s.db.WithContext(ctx), apiKeyID)
if err != nil {
return nil, err
}
- s.apiKeyCache.Set(apiKeyID, permissions)
+ s.cacheFillMu.Lock()
+ if s.apiKeyCacheGen.Load() == gen {
+ s.apiKeyCache.Set(apiKeyID, permissions)
+ }
+ s.cacheFillMu.Unlock()
return permissions, nil
}
@@ -687,7 +707,7 @@ func (s *RoleService) SetApiKeyPermissions(ctx context.Context, apiKeyID string,
if err != nil {
return err
}
- s.apiKeyCache.Delete(apiKeyID)
+ s.InvalidateApiKey(apiKeyID)
return nil
}
@@ -913,31 +933,20 @@ func (s *RoleService) ReconcileEnvOidcMappings(ctx context.Context, rawSpec stri
// auth_service after a login that mutates assignments, and from any mutation
// path that doesn't already invalidate explicitly.
func (s *RoleService) InvalidateUser(userID string) {
+ s.cacheFillMu.Lock()
+ defer s.cacheFillMu.Unlock()
+ s.userCacheGen.Add(1)
s.userCache.Delete(userID)
}
// InvalidateApiKey drops the cached PermissionSet for one API key.
func (s *RoleService) InvalidateApiKey(apiKeyID string) {
+ s.cacheFillMu.Lock()
+ defer s.cacheFillMu.Unlock()
+ s.apiKeyCacheGen.Add(1)
s.apiKeyCache.Delete(apiKeyID)
}
-// invalidateUsersAssignedToInternal invalidates every user holding an assignment to
-// the given role. Called after a role's permissions change.
-func (s *RoleService) invalidateUsersAssignedToInternal(ctx context.Context, roleID string) {
- var userIDs []string
- if err := s.db.WithContext(ctx).
- Model(&UserRoleAssignment{}).
- Distinct("user_id").
- Where("role_id = ?", roleID).
- Pluck("user_id", &userIDs).Error; err != nil {
- slog.WarnContext(ctx, "failed to collect users for cache invalidation", "error", err, "role_id", roleID)
- return
- }
- for _, id := range userIDs {
- s.userCache.Delete(id)
- }
-}
-
// ---------- helpers ----------
func validatePermissionsInternal(perms []string) error {
diff --git a/backend/internal/template/service.go b/backend/internal/template/service.go
index 26c367e35..8f3591437 100644
--- a/backend/internal/template/service.go
+++ b/backend/internal/template/service.go
@@ -13,6 +13,7 @@ import (
"path/filepath"
"strings"
"sync"
+ "sync/atomic"
"time"
"uuid"
@@ -47,7 +48,8 @@ type TemplateService struct {
lookupIP httputils.LookupIPFunc
settingsService *settings.SettingsService
- remoteCache *hot.HotCache[struct{}, []ComposeTemplate]
+ remoteCache *hot.HotCache[uint64, []ComposeTemplate]
+ remoteGeneration atomic.Uint64
registryMu sync.RWMutex
registryFetchMeta map[string]*registryFetchMeta
@@ -65,7 +67,7 @@ const (
remoteIDPrefix = "remote"
)
-var errNoRemoteTemplates = errors.New("remote template registries returned no templates")
+var errRemoteCacheInvalidatedInternal = errors.New("remote template cache invalidated")
func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.Client, settingsService *settings.SettingsService) *TemplateService {
if httpClient == nil {
@@ -82,19 +84,20 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.C
}
service.safeHTTPClient = service.newSafeHTTPClientInternal()
revalidationCtx := context.WithoutCancel(ctx)
- loader := func(_ []struct{}) (map[struct{}][]ComposeTemplate, error) {
+ loader := func(generations []uint64) (map[uint64][]ComposeTemplate, error) {
loadCtx, cancel := context.WithTimeout(revalidationCtx, 2*time.Minute)
defer cancel()
- templates, err := service.loadRemoteTemplates(loadCtx)
- if err != nil {
- return nil, err
- }
- if len(templates) == 0 {
- return nil, errNoRemoteTemplates
+ catalogs := make(map[uint64][]ComposeTemplate, len(generations))
+ for _, generation := range generations {
+ templates, err := service.loadRemoteTemplatesInternal(loadCtx, generation)
+ if err != nil {
+ return nil, err
+ }
+ catalogs[generation] = templates
}
- return map[struct{}][]ComposeTemplate{{}: templates}, nil
+ return catalogs, nil
}
- service.remoteCache = hot.NewHotCache[struct{}, []ComposeTemplate](hot.LRU, 1).
+ service.remoteCache = hot.NewHotCache[uint64, []ComposeTemplate](hot.LRU, 1).
WithTTL(remoteCacheDuration).
WithLoaders(loader).
WithRevalidation(24*time.Hour, loader).
@@ -125,39 +128,47 @@ func (s *TemplateService) getTemplatesDirectoryInternal(ctx context.Context) (st
return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSettingInternal(ctx)))
}
-func (s *TemplateService) ensureRemoteTemplatesLoaded(_ context.Context) error {
+func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh bool) ([]ComposeTemplate, error) {
if s.remoteCache == nil {
- return errors.New("remote template cache is not initialized")
+ return nil, errors.New("remote template cache is not initialized")
}
- templates, found, err := s.remoteCache.Get(struct{}{})
- if err != nil {
- return fmt.Errorf("failed to load remote templates: %w", err)
+ for {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ generation := s.remoteGeneration.Load()
+ var templates []ComposeTemplate
+ var err error
+ if refresh {
+ templates, err = s.loadRemoteTemplatesInternal(ctx, generation)
+ s.registryMu.Lock()
+ if generation == s.remoteGeneration.Load() && err == nil {
+ s.remoteCache.Set(generation, templates)
+ }
+ s.registryMu.Unlock()
+ } else {
+ var found bool
+ templates, found, err = s.remoteCache.Get(generation)
+ if err == nil && !found {
+ err = errors.New("remote template catalog not found")
+ }
+ }
+ if generation != s.remoteGeneration.Load() {
+ continue
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to load remote templates: %w", err)
+ }
+ return templates, nil
}
- return kit.Ternary(!found || len(templates) == 0, errNoRemoteTemplates, nil)
-}
-
-func (s *TemplateService) refreshRemoteTemplates(ctx context.Context) error {
- templates, err := s.loadRemoteTemplates(ctx)
- if err != nil {
- return fmt.Errorf("failed to load remote templates: %w", err)
- }
-
- if len(templates) == 0 {
- return errNoRemoteTemplates
- }
- if s.remoteCache == nil {
- return errors.New("remote template cache is not initialized")
- }
- s.remoteCache.Set(struct{}{}, templates)
- return nil
}
func (s *TemplateService) GetAllTemplates(ctx context.Context) ([]ComposeTemplate, error) {
- return s.getMergedTemplates(ctx)
+ return s.getMergedTemplatesInternal(ctx)
}
func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params pagination.QueryParams) ([]tmpl.Template, pagination.Response, error) {
- templates, err := s.getMergedTemplates(ctx)
+ templates, err := s.getMergedTemplatesInternal(ctx)
if err != nil {
return nil, pagination.Response{}, err
}
@@ -235,11 +246,12 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeT
return nil, fmt.Errorf("failed to query local template: %w", err)
}
- if err := s.ensureRemoteTemplatesLoaded(ctx); err != nil && !errors.Is(err, errNoRemoteTemplates) {
+ templates, err := s.remoteTemplatesInternal(ctx, false)
+ if err != nil {
return nil, fmt.Errorf("template %q lookup failed: registry refresh error: %w", id, err)
}
- if found := s.lookupRemoteFromCacheInternal(id); found != nil {
+ if found := s.lookupRemoteTemplateInternal(templates, id); found != nil {
return found, nil
}
@@ -247,27 +259,21 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeT
// before we return "not found" — the cache may be stale or the previous refresh
// silently returned empty.
if strings.HasPrefix(id, remoteIDPrefix+":") {
- slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateID", id, "cacheSize", s.remoteCacheSizeInternal())
- if refreshErr := s.refreshRemoteTemplates(ctx); refreshErr != nil && !errors.Is(refreshErr, errNoRemoteTemplates) {
+ slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateID", id, "cacheSize", len(templates))
+ templates, refreshErr := s.remoteTemplatesInternal(ctx, true)
+ if refreshErr != nil {
return nil, fmt.Errorf("template %q not found and registry refresh failed: %w", id, refreshErr)
}
- if found := s.lookupRemoteFromCacheInternal(id); found != nil {
+ if found := s.lookupRemoteTemplateInternal(templates, id); found != nil {
return found, nil
}
- return nil, common.Classify(common.ErrTemplateNotFound, fmt.Errorf("Template not found: %w", fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, s.remoteCacheSizeInternal()))) //nolint:staticcheck // Preserve the existing error message.
+ return nil, common.Classify(common.ErrTemplateNotFound, fmt.Errorf("Template not found: %w", fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, len(templates)))) //nolint:staticcheck // Preserve the existing error message.
}
return nil, common.Classify(common.ErrTemplateNotFound, errors.New("Template not found")) //nolint:staticcheck // Preserve the existing error message.
}
-func (s *TemplateService) lookupRemoteFromCacheInternal(id string) *ComposeTemplate {
- if s.remoteCache == nil {
- return nil
- }
- templates, found := s.remoteCache.Peek(struct{}{})
- if !found {
- return nil
- }
+func (s *TemplateService) lookupRemoteTemplateInternal(templates []ComposeTemplate, id string) *ComposeTemplate {
for i := range templates {
if templates[i].ID == id {
cloned := cloneRemoteTemplates(templates[i : i+1])
@@ -277,14 +283,6 @@ func (s *TemplateService) lookupRemoteFromCacheInternal(id string) *ComposeTempl
return nil
}
-func (s *TemplateService) remoteCacheSizeInternal() int {
- if s.remoteCache == nil {
- return 0
- }
- templates, _ := s.remoteCache.Peek(struct{}{})
- return len(templates)
-}
-
func (s *TemplateService) CreateTemplate(ctx context.Context, template *ComposeTemplate) error {
if err := normalization.Normalize(template); err != nil {
return err
@@ -491,7 +489,7 @@ func (s *TemplateService) CreateRegistry(ctx context.Context, registry *Template
return err
}
- s.invalidateRemoteCache()
+ s.invalidateRemoteCacheInternal()
return nil
}
@@ -526,7 +524,7 @@ func (s *TemplateService) UpdateRegistry(ctx context.Context, id string, updates
return err
}
- s.invalidateRemoteCache()
+ s.invalidateRemoteCacheInternal()
return nil
}
@@ -568,19 +566,21 @@ func (s *TemplateService) DeleteRegistry(ctx context.Context, id string) error {
return err
}
- s.invalidateRemoteCache()
+ s.invalidateRemoteCacheInternal()
return nil
}
-func (s *TemplateService) loadRemoteTemplates(ctx context.Context) ([]ComposeTemplate, error) {
+func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, generation uint64) ([]ComposeTemplate, error) {
registries, err := s.GetRegistries(ctx)
if err != nil {
return nil, err
}
var (
- mu sync.Mutex
- templates []ComposeTemplate
+ mu sync.Mutex
+ templates []ComposeTemplate
+ fetchErrors []error
+ successfulRegistries int
)
g, groupCtx := errgroup.WithContext(ctx)
@@ -594,21 +594,29 @@ func (s *TemplateService) loadRemoteTemplates(ctx context.Context) ([]ComposeTem
g.Go(func() (workerErr error) {
defer utils.RecoverToError(&workerErr, "template worker")
- remoteTemplates, err := s.fetchRegistryTemplates(groupCtx, &reg)
+ remoteTemplates, err := s.fetchRegistryTemplatesInternal(groupCtx, &reg, generation)
if err != nil {
slog.WarnContext(groupCtx, "failed to fetch templates from registry", "registry", reg.Name, "url", reg.URL, "error", err)
s.registryMu.Lock()
- s.registryErrors[reg.ID] = err.Error()
+ if generation == s.remoteGeneration.Load() {
+ s.registryErrors[reg.ID] = err.Error()
+ }
s.registryMu.Unlock()
+ mu.Lock()
+ fetchErrors = append(fetchErrors, fmt.Errorf("registry %q: %w", reg.Name, err))
+ mu.Unlock()
return nil // Don't fail the whole group if one registry fails
}
s.registryMu.Lock()
- delete(s.registryErrors, reg.ID)
+ if generation == s.remoteGeneration.Load() {
+ delete(s.registryErrors, reg.ID)
+ }
s.registryMu.Unlock()
mu.Lock()
defer mu.Unlock()
+ successfulRegistries++
for _, template := range remoteTemplates {
template.Registry = cloneRegistry(&reg)
template.RegistryID = mo.EmptyableToOption(strings.TrimSpace(reg.ID)).ToPointer()
@@ -621,6 +629,12 @@ func (s *TemplateService) loadRemoteTemplates(ctx context.Context) ([]ComposeTem
if err := g.Wait(); err != nil {
return nil, err
}
+ if generation != s.remoteGeneration.Load() {
+ return nil, errRemoteCacheInvalidatedInternal
+ }
+ if successfulRegistries == 0 && len(fetchErrors) > 0 {
+ return nil, errors.Join(fetchErrors...)
+ }
return templates, nil
}
@@ -651,10 +665,14 @@ func (s *TemplateService) doGET(ctx context.Context, url string) ([]byte, error)
return body, nil
}
-// fetchRegistryTemplates performs a conditional GET using If-Modified-Since.
+// fetchRegistryTemplatesInternal performs a conditional GET using If-Modified-Since.
// If the server replies 304 Not Modified, cached templates for the registry are reused.
-func (s *TemplateService) fetchRegistryTemplates(ctx context.Context, reg *TemplateRegistry) ([]ComposeTemplate, error) {
+func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, reg *TemplateRegistry, generation uint64) ([]ComposeTemplate, error) {
s.registryMu.RLock()
+ if generation != s.remoteGeneration.Load() {
+ s.registryMu.RUnlock()
+ return nil, errRemoteCacheInvalidatedInternal
+ }
fetchMeta := s.registryFetchMeta[reg.ID]
s.registryMu.RUnlock()
@@ -704,6 +722,10 @@ func (s *TemplateService) fetchRegistryTemplates(ctx context.Context, reg *Templ
Templates: cloneRemoteTemplates(templates),
}
s.registryMu.Lock()
+ if generation != s.remoteGeneration.Load() {
+ s.registryMu.Unlock()
+ return nil, errRemoteCacheInvalidatedInternal
+ }
s.registryFetchMeta[reg.ID] = newMeta
s.registryMu.Unlock()
@@ -1001,14 +1023,15 @@ func cloneRegistry(registry *TemplateRegistry) *TemplateRegistry {
return new(*registry)
}
-func (s *TemplateService) invalidateRemoteCache() {
+func (s *TemplateService) invalidateRemoteCacheInternal() {
+ s.registryMu.Lock()
+ defer s.registryMu.Unlock()
+ s.remoteGeneration.Add(1)
+ s.registryFetchMeta = make(map[string]*registryFetchMeta)
+ s.registryErrors = make(map[string]string)
if s.remoteCache != nil {
s.remoteCache.Purge()
}
-
- s.registryMu.Lock()
- s.registryFetchMeta = make(map[string]*registryFetchMeta)
- s.registryMu.Unlock()
}
func (s *TemplateService) SyncLocalTemplatesFromFilesystem(ctx context.Context) error {
@@ -1180,7 +1203,7 @@ func (s *TemplateService) GetTemplateContentWithParsedData(ctx context.Context,
}, nil
}
-func (s *TemplateService) getMergedTemplates(ctx context.Context) ([]ComposeTemplate, error) {
+func (s *TemplateService) getMergedTemplatesInternal(ctx context.Context) ([]ComposeTemplate, error) {
if err := s.syncFilesystemTemplatesInternal(ctx); err != nil {
slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)
}
@@ -1191,10 +1214,10 @@ func (s *TemplateService) getMergedTemplates(ctx context.Context) ([]ComposeTemp
return nil, fmt.Errorf("failed to get local templates: %w", err)
}
- if err := s.ensureRemoteTemplatesLoaded(ctx); err != nil {
+ remoteTemplates, err := s.remoteTemplatesInternal(ctx, false)
+ if err != nil {
slog.WarnContext(ctx, "failed to load remote templates", "error", err)
} else {
- remoteTemplates, _ := s.remoteCache.Peek(struct{}{})
copied := cloneRemoteTemplates(remoteTemplates)
if len(copied) > 0 {
diff --git a/backend/internal/template/service_test.go b/backend/internal/template/service_test.go
index 20a60bc65..69f805cb0 100644
--- a/backend/internal/template/service_test.go
+++ b/backend/internal/template/service_test.go
@@ -151,7 +151,7 @@ services:
Enabled: true,
}
- templates, err := service.fetchRegistryTemplates(context.Background(), registry)
+ templates, err := service.fetchRegistryTemplatesInternal(context.Background(), registry, service.remoteGeneration.Load())
require.NoError(t, err)
require.Len(t, templates, 2)
require.NotNil(t, templates[0].Metadata)
@@ -160,7 +160,7 @@ services:
require.Nil(t, templates[1].Metadata.IconURL)
require.EqualValues(t, 1, composeHits.Load())
- cachedTemplates, err := service.fetchRegistryTemplates(context.Background(), registry)
+ cachedTemplates, err := service.fetchRegistryTemplatesInternal(context.Background(), registry, service.remoteGeneration.Load())
require.NoError(t, err)
require.Len(t, cachedTemplates, 2)
require.NotNil(t, cachedTemplates[0].Metadata)
@@ -259,7 +259,7 @@ func TestGetAllTemplatesPaginated_FiltersByType(t *testing.T) {
require.NoError(t, db.WithContext(context.Background()).Create(&localTemplates).Error)
service := NewTemplateService(context.Background(), db, http.DefaultClient, nil)
- service.remoteCache.Set(struct{}{}, []ComposeTemplate{
+ service.remoteCache.Set(service.remoteGeneration.Load(), []ComposeTemplate{
{
ID: "remote-one", CreatedAt: now, UpdatedAt: &now,
Name: "Remote One",
diff --git a/backend/internal/volume/helper_container.go b/backend/internal/volume/helper_container.go
index c14d1012e..84dc80ea7 100644
--- a/backend/internal/volume/helper_container.go
+++ b/backend/internal/volume/helper_container.go
@@ -12,6 +12,7 @@ import (
"strings"
"time"
+ cerrdefs "github.com/containerd/errdefs"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
@@ -175,7 +176,11 @@ func (s *VolumeService) acquireVolumeHelperInternal(ctx context.Context, volumeN
// creation itself can pull an image, so it must not run under helperMu.
func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (string, error) {
resultCh := s.helperGroup.DoChan(volumeName, func() (any, error) {
- if containerID, ok := s.getReusableHelperInternal(ctx, dockerClient, volumeName).Get(); ok {
+ helper, err := s.getReusableHelperInternal(ctx, dockerClient, volumeName)
+ if err != nil {
+ return nil, err
+ }
+ if containerID, ok := helper.Get(); ok {
return containerID, nil
}
@@ -274,8 +279,13 @@ func (s *VolumeService) startHelperContainerInternal(ctx context.Context, docker
}
if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- return "", nil, fmt.Errorf("failed to start temp container: %w", err)
+ startErr := fmt.Errorf("failed to start temp container: %w", err)
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)
+ defer cancel()
+ if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !cerrdefs.IsNotFound(cleanupErr) {
+ return "", nil, errors.Join(startErr, fmt.Errorf("failed to remove temp container after start failure: %w", cleanupErr))
+ }
+ return "", nil, startErr
}
cleanup := func() {
@@ -285,25 +295,38 @@ func (s *VolumeService) startHelperContainerInternal(ctx context.Context, docker
return resp.ID, cleanup, nil
}
-func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) mo.Option[string] {
+func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (mo.Option[string], error) {
s.helperMu.Lock()
helper := s.helperByVolume[volumeName]
s.helperMu.Unlock()
if helper == nil || helper.id == "" {
- return mo.None[string]()
+ return mo.None[string](), nil
}
inspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, helper.id, client.ContainerInspectOptions{})
+ if err != nil && !cerrdefs.IsNotFound(err) {
+ return mo.None[string](), fmt.Errorf("failed to inspect volume helper container: %w", err)
+ }
+ if err == nil && (inspect.Container.State == nil || !inspect.Container.State.Running) {
+ removeCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI)
+ _, removeErr := dockerClient.ContainerRemove(removeCtx, helper.id, volumehelper.RemoveOptions())
+ cancel()
+ if removeErr != nil && !cerrdefs.IsNotFound(removeErr) {
+ return mo.None[string](), fmt.Errorf("failed to remove stopped volume helper container: %w", removeErr)
+ }
+ }
if err != nil || inspect.Container.State == nil || !inspect.Container.State.Running {
s.helperMu.Lock()
- delete(s.helperByVolume, volumeName)
+ if s.helperByVolume[volumeName] == helper {
+ delete(s.helperByVolume, volumeName)
+ }
s.helperMu.Unlock()
- return mo.None[string]()
+ return mo.None[string](), nil
}
s.touchHelperInternal(volumeName)
- return mo.Some(helper.id)
+ return mo.Some(helper.id), nil
}
// touchHelperInternal records that the helper for volumeName just serviced a
diff --git a/backend/internal/volume/workspace.go b/backend/internal/volume/workspace.go
index 1f75d01b5..9ff636e2a 100644
--- a/backend/internal/volume/workspace.go
+++ b/backend/internal/volume/workspace.go
@@ -25,6 +25,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/event"
dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
acfsutils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/acfs"
workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"
@@ -83,11 +84,16 @@ func (s *VolumeService) readVolumeWorkspaceFromContainerInternal(ctx context.Con
maxEntries = 10000
}
- pipeReader, pipeWriter := io.Pipe()
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
return nil, err
}
+ workCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ pipeReader, pipeWriter := io.Pipe()
+ // io.PipeReader.CloseWithError always returns nil.
+ stopClose := context.AfterFunc(workCtx, func() { _ = pipeReader.CloseWithError(workCtx.Err()) })
+ defer stopClose()
var stderr bytes.Buffer
done := make(chan error, 1)
go func() {
@@ -95,7 +101,7 @@ func (s *VolumeService) readVolumeWorkspaceFromContainerInternal(ctx context.Con
"acfs", "walk", "--root", "/volume", "--path", "/",
"--max-depth", strconv.Itoa(maxDepth), "--max-entries", strconv.Itoa(maxEntries),
}
- exitCode, execErr := dockerutil.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{
+ exitCode, execErr := dockerutil.ExecInContainer(workCtx, dockerClient, containerID, client.ExecCreateOptions{
AttachStdout: true,
AttachStderr: true,
Cmd: cmd,
@@ -109,11 +115,12 @@ func (s *VolumeService) readVolumeWorkspaceFromContainerInternal(ctx context.Con
workspace, parseErr := decodeVolumeWorkspaceWalkInternal(pipeReader, maxEntries, s.volumeWorkspaceMaxFileSizeBytesInternal())
if parseErr != nil {
+ cancel()
_ = pipeReader.CloseWithError(parseErr)
}
execErr := <-done
_ = pipeReader.Close()
- if execErr != nil {
+ if execErr != nil && (parseErr == nil || ctx.Err() != nil || !errors.Is(execErr, context.Canceled)) {
return nil, classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace")
}
if parseErr != nil {
@@ -314,6 +321,9 @@ func (s *VolumeService) DownloadVolumeWorkspaceFile(ctx context.Context, volumeN
}
type volumeWorkspaceReadStreamInternal struct {
+ ctx context.Context
+ cancel context.CancelFunc
+ stopClose func() bool
pipe *io.PipeReader
done <-chan error
cleanup func()
@@ -351,20 +361,28 @@ func (r *volumeWorkspaceReadStreamInternal) Read(buffer []byte) (int, error) {
func (r *volumeWorkspaceReadStreamInternal) Close() error {
r.once.Do(func() {
+ r.stopClose()
+ r.cancel()
_ = r.pipe.Close()
r.err = <-r.done
+ if r.ctx.Err() == nil && errors.Is(r.err, context.Canceled) {
+ r.err = nil
+ }
r.cleanup()
})
return r.err
}
func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, containerID, relativePath string, limit int64, cleanup func()) (io.ReadCloser, int64, error) {
- pipeReader, pipeWriter := io.Pipe()
dockerClient, err := s.dockerService.GetClient(ctx)
if err != nil {
cleanup()
return nil, 0, err
}
+ workCtx, cancel := context.WithCancel(ctx)
+ pipeReader, pipeWriter := io.Pipe()
+ // io.PipeReader.CloseWithError always returns nil.
+ stopClose := context.AfterFunc(workCtx, func() { _ = pipeReader.CloseWithError(workCtx.Err()) })
done := make(chan error, 1)
cmd := []string{"acfs", "read", "--root", "/volume", "--path", "/" + relativePath}
if limit > 0 {
@@ -372,7 +390,7 @@ func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, co
}
go func() {
var stderr bytes.Buffer
- exitCode, execErr := dockerutil.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{
+ exitCode, execErr := dockerutil.ExecInContainer(workCtx, dockerClient, containerID, client.ExecCreateOptions{
AttachStdout: true,
AttachStderr: true,
Cmd: cmd,
@@ -383,21 +401,24 @@ func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, co
if execErr != nil {
execErr = classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace file")
}
+ stopClose()
_ = pipeWriter.CloseWithError(execErr)
done <- execErr
}()
payloadSize, err := acfs.ReadStreamHeader(pipeReader)
if err != nil {
+ cancel()
_ = pipeReader.CloseWithError(err)
execErr := <-done
cleanup()
- if execErr != nil {
+ if execErr != nil && (ctx.Err() != nil || !errors.Is(execErr, context.Canceled)) {
return nil, 0, execErr
}
return nil, 0, fmt.Errorf("parse volume workspace read header: %w", err)
}
if payloadSize > uint64(1<<63-1) {
+ cancel()
_ = pipeReader.Close()
<-done
cleanup()
@@ -405,6 +426,9 @@ func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, co
}
size := int64(payloadSize)
return &volumeWorkspaceReadStreamInternal{
+ ctx: ctx,
+ cancel: cancel,
+ stopClose: stopClose,
pipe: pipeReader,
done: done,
cleanup: cleanup,
@@ -1049,8 +1073,13 @@ func (s *VolumeService) createVolumeWorkspaceMutationContainerInternal(ctx conte
return "", nil, fmt.Errorf("create volume workspace helper: %w", err)
}
if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {
- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())
- return "", nil, fmt.Errorf("start volume workspace helper: %w", err)
+ startErr := fmt.Errorf("start volume workspace helper: %w", err)
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)
+ defer cancel()
+ if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !cerrdefs.IsNotFound(cleanupErr) {
+ return "", nil, errors.Join(startErr, fmt.Errorf("remove volume workspace helper after start failure: %w", cleanupErr))
+ }
+ return "", nil, startErr
}
return resp.ID, func() {
_, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())
diff --git a/backend/pkg/dockerutil/exec.go b/backend/pkg/dockerutil/exec.go
index 817da5d7a..988290b0f 100644
--- a/backend/pkg/dockerutil/exec.go
+++ b/backend/pkg/dockerutil/exec.go
@@ -63,10 +63,16 @@ func ExecInContainer(ctx context.Context, dockerClient *client.Client, container
return 0, fmt.Errorf("failed to attach to exec: %w", err)
}
defer attachResp.Close()
+ stopClose := context.AfterFunc(ctx, attachResp.Close)
+ defer stopClose()
- if err := WaitStdCopy(StartStdCopy(attachResp.Reader, stdout, stderr)); err != nil {
+ copyErr := WaitStdCopy(StartStdCopy(attachResp.Reader, stdout, stderr))
+ if err := ctx.Err(); err != nil {
return 0, fmt.Errorf("failed to read exec output: %w", err)
}
+ if copyErr != nil {
+ return 0, fmt.Errorf("failed to read exec output: %w", copyErr)
+ }
inspect, err := dockerClient.ExecInspect(ctx, execResp.ID, client.ExecInspectOptions{})
if err != nil {
diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.go
index dee21bdbd..c29858acb 100644
--- a/backend/pkg/fswatch/watcher.go
+++ b/backend/pkg/fswatch/watcher.go
@@ -30,6 +30,7 @@ type Watcher struct {
debounce time.Duration
stopCh chan struct{}
stoppedCh chan struct{}
+ cancel context.CancelFunc
watchAliases map[string]string
pendingPaths map[string]struct{}
mu sync.Mutex
@@ -105,7 +106,9 @@ func (fw *Watcher) Start(ctx context.Context) error {
"error", err)
}
- go fw.watchLoop(ctx)
+ watchCtx, cancel := context.WithCancel(ctx)
+ fw.cancel = cancel
+ go fw.watchLoopInternal(watchCtx)
fw.started = true
slog.InfoContext(ctx, "Filesystem watcher started", "path", fw.watchedPath)
@@ -125,9 +128,11 @@ func (fw *Watcher) StopWatching() error {
fw.mu.Lock()
fw.stopped = true
started := fw.started
+ cancel := fw.cancel
fw.mu.Unlock()
if started {
+ cancel()
close(fw.stopCh)
<-fw.stoppedCh // Wait for watchLoop to finish
}
@@ -145,7 +150,7 @@ func (fw *Watcher) StopWatching() error {
return fw.stopErr
}
-func (fw *Watcher) watchLoop(ctx context.Context) {
+func (fw *Watcher) watchLoopInternal(ctx context.Context) {
defer close(fw.stoppedCh)
debounceTimer := time.NewTimer(fw.debounce)
@@ -221,18 +226,31 @@ func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bo
"goroutines", runtime.NumGoroutine())
*lastGoroutineLog = time.Now()
}
- if fw.onChange != nil {
- fw.callbacks.Go(func() { fw.onChange(ctx) })
- }
+ var paths []string
if fw.onChangePaths != nil {
- paths := fw.drainPendingPathsInternal()
- if len(paths) > 0 {
- fw.callbacks.Go(func() { fw.onChangePaths(ctx, paths) })
- }
+ paths = fw.drainPendingPathsInternal()
}
+ fw.notifyChangesInternal(ctx, paths)
return false
}
+func (fw *Watcher) notifyChangesInternal(ctx context.Context, paths []string) {
+ if fw.onChange != nil {
+ fw.callbacks.Go(func() {
+ if ctx.Err() == nil {
+ fw.onChange(ctx)
+ }
+ })
+ }
+ if fw.onChangePaths != nil && len(paths) > 0 {
+ fw.callbacks.Go(func() {
+ if ctx.Err() == nil {
+ fw.onChangePaths(ctx, paths)
+ }
+ })
+ }
+}
+
func (fw *Watcher) recordPendingPathInternal(path string) {
if path == "" {
return
@@ -486,12 +504,7 @@ func (fw *Watcher) reconnectInternal(ctx context.Context) bool {
slog.WarnContext(ctx, "Some filesystem subscriptions could not be restored", "error", err)
}
// Changes while disconnected need a full reconciliation.
- if fw.onChange != nil {
- fw.callbacks.Go(func() { fw.onChange(ctx) })
- }
- if fw.onChangePaths != nil {
- fw.callbacks.Go(func() { fw.onChangePaths(ctx, []string{fw.watchedPath}) })
- }
+ fw.notifyChangesInternal(ctx, []string{fw.watchedPath})
return true
}
if closeErr := watcher.Close(); closeErr != nil {
diff --git a/backend/pkg/libarcane/rustic/run.go b/backend/pkg/libarcane/rustic/run.go
index 24e0c61ab..a405e9d66 100644
--- a/backend/pkg/libarcane/rustic/run.go
+++ b/backend/pkg/libarcane/rustic/run.go
@@ -5,8 +5,12 @@ import (
"cmp"
"context"
"fmt"
+ "log/slog"
"strings"
+ "time"
+ cerrdefs "github.com/containerd/errdefs"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"
"github.com/moby/moby/api/pkg/stdcopy"
"github.com/moby/moby/api/types/container"
@@ -49,7 +53,19 @@ func Run(ctx context.Context, dockerClient *client.Client, password string, comm
return "", fmt.Errorf("failed to create Rustic container: %w", err)
}
defer func() {
- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), created.ID, volumehelper.RemoveOptions())
+ // Keep caller locks and stopped consumers protected until removal is confirmed.
+ // Request cancellation cannot prove that Rustic has stopped writing.
+ cleanupCtx := context.WithoutCancel(ctx)
+ for {
+ attemptCtx, cancel := context.WithTimeout(cleanupCtx, timeouts.DefaultDockerAPI)
+ _, err := dockerClient.ContainerRemove(attemptCtx, created.ID, volumehelper.RemoveOptions())
+ cancel()
+ if err == nil || cerrdefs.IsNotFound(err) {
+ return
+ }
+ slog.WarnContext(cleanupCtx, "failed to remove Rustic container, retrying", "container_id", created.ID, "error", err)
+ time.Sleep(5 * time.Second)
+ }
}()
if _, err := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {
return "", fmt.Errorf("failed to start Rustic container: %w", err)
diff --git a/go.work.sum b/go.work.sum
index 0d92bbde4..5833d5349 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -383,6 +383,11 @@ cyphar.com/go-pathrs v0.2.4/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcG
cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik=
cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY=
+emperror.dev/emperror v0.33.0 h1:urYop6KLYxKVpZbt9ADC4eVG3WDnJFE6Ye3j07wUu/I=
+emperror.dev/emperror v0.33.0/go.mod h1:CeOIKPcppTE8wn+3xBNcdzdHMMIP77sLOHS0Ik56m+w=
+emperror.dev/errors v0.8.0/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE=
+emperror.dev/errors v0.8.1 h1:UavXZ5cSX/4u9iyvH6aDcuGkVjeexUGJ7Ij7G4VfQT0=
+emperror.dev/errors v0.8.1/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
@@ -2763,8 +2768,6 @@ go.getarcane.app/kit v0.1.0 h1:82oO/i1fS4xbzAG4Xc8ALOWOwW4kwQYDdBpVHE19QkI=
go.getarcane.app/kit v0.1.0/go.mod h1:oiH4hLowxfQYFtmldyk/y79XYBHYfzgd86MrSQS9UEg=
go.getarcane.app/kit v0.2.0 h1:3PPHo5nHuns1vjqmCMFPFdnmbH/qQOserONCba/gBi4=
go.getarcane.app/kit v0.2.0/go.mod h1:TGqQCbAJs3ldzkFncXZ93wTVSDhAMQlqyNdVO2CNjIk=
-go.getarcane.app/kit v0.3.2 h1:L7CAvGawDg70K/vVxDeM2xZBCHqwuthc9B5OwQwL0Hc=
-go.getarcane.app/kit v0.3.2/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=
go.getarcane.app/streams v0.4.0 h1:MtBX1SVgt3nrcRNr5SYAvckGp9a5qeSjMTw/LAGwlzk=
go.getarcane.app/streams v0.4.0/go.mod h1:LHMKblQvsL1nqKh/Y+KgQbwFSMI2I7bs9u8ZwZqzjnQ=
go.getarcane.app/streams v0.4.2 h1:+xrRHiOt25cLbFTDpPLzv7eFkk7biwDz5YjBXkbWUPw=
@@ -2936,6 +2939,7 @@ go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp
go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
@@ -2943,6 +2947,7 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=