Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix(backend): prevent stale cache fills and harden stream cleanup (#4267)
Kyle Mendellb48b11fdf738
diff --git a/backend/api/ws/container_stats.go b/backend/api/ws/container_stats.goindex f11d372d6..67804ad58 100644--- a/backend/api/ws/container_stats.go+++ b/backend/api/ws/container_stats.go@@ -102,9 +102,7 @@ func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hubif cleanupTimer != timer {return}- if existing, ok := h.containerStatsHubs.Load(containerID); ok && existing == hub {- h.containerStatsHubs.Delete(containerID)- }+ h.containerStatsHubs.CompareAndDelete(containerID, hub)slog.Debug("container stats hub idle, cleaning up upstream stream", "containerID", containerID)cleanupTimer = nilcancel()@@ -128,7 +126,9 @@ func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hubdefer close(statsChan)err := h.containerService.StreamStats(ctx, containerID, statsChan)+ h.containerStatsHubs.CompareAndDelete(containerID, hub)if err == nil || errors.Is(err, context.Canceled) || ctx.Err() != nil {+ cancel()return}@@ -143,7 +143,6 @@ func (h *WebSocketHandler) runContainerStatsHubInternal(containerID string, hub}); marshalErr == nil {hub.Broadcast(b)}- h.containerStatsHubs.CompareAndDelete(containerID, hub)// Cancelling immediately would race hub.Run between ctx.Done and the// queued error frame, dropping the very message clients need. Give thediff --git a/backend/api/ws/handler.go b/backend/api/ws/handler.goindex fba2911ae..2ba41e64a 100644--- a/backend/api/ws/handler.go+++ b/backend/api/ws/handler.go@@ -32,18 +32,19 @@ var defaultWebSocketMetrics = wshub.NewWebSocketMetrics()// WebSocketHandler consolidates all WebSocket and streaming endpoints.// REST endpoints are handled by Huma handlers.type WebSocketHandler struct {- projectService *project.ProjectService- containerService *container.ContainerService- swarmService *swarm.SwarmService- systemService *system.SystemService- diagnosticsService *diagnostics.DiagnosticsService- checkWSOrigin func(*http.Request) bool- wsMetrics *wshub.WebSocketMetrics- activeConnections sync.Map- logStreamsMu sync.Mutex- logStreams map[string]*wsLogStream- cpuCache concurrency.Snapshot[float64]- systemStaticInfo struct {+ projectService *project.ProjectService+ containerService *container.ContainerService+ swarmService *swarm.SwarmService+ systemService *system.SystemService+ diagnosticsService *diagnostics.DiagnosticsService+ checkWSOrigin func(*http.Request) bool+ wsMetrics *wshub.WebSocketMetrics+ activeConnectionsMu sync.Mutex+ activeConnections map[string]int+ logStreamsMu sync.Mutex+ logStreams map[string]*wsLogStream+ cpuCache concurrency.Snapshot[float64]+ systemStaticInfo struct {once sync.OncecpuCount inthostname stringdiff --git a/backend/api/ws/system_stats.go b/backend/api/ws/system_stats.goindex 69afaa0d9..a25f6df73 100644--- a/backend/api/ws/system_stats.go+++ b/backend/api/ws/system_stats.go@@ -7,7 +7,6 @@ import ("net/http""runtime""sync"- "sync/atomic""time""github.com/coder/websocket"@@ -37,30 +36,27 @@ const (// ============================================================================// checkRateLimitInternal checks and applies rate limiting for WebSocket connections.-// Returns the counter and whether the connection should be allowed.-func (h *WebSocketHandler) checkRateLimitInternal(clientIP string) (*int32, bool) {- connCount, _ := h.activeConnections.LoadOrStore(clientIP, new(int32))- count, ok := connCount.(*int32)- if !ok {- return nil, false+func (h *WebSocketHandler) checkRateLimitInternal(clientIP string) bool {+ h.activeConnectionsMu.Lock()+ defer h.activeConnectionsMu.Unlock()+ if h.activeConnections[clientIP] >= 5 {+ return false}-- currentCount := atomic.AddInt32(count, 1)- if currentCount > 5 {- atomic.AddInt32(count, -1)- return nil, false+ if h.activeConnections == nil {+ h.activeConnections = make(map[string]int)}- return count, true+ h.activeConnections[clientIP]+++ return true}// releaseRateLimitInternal decrements the connection counter and cleans up if needed.-func (h *WebSocketHandler) releaseRateLimitInternal(clientIP string, count *int32) {- newCount := atomic.AddInt32(count, -1)- if newCount <= 0 {- // CompareAndDelete, not Delete: a plain delete removed whatever counter- // was stored for this IP, which after a racing reconnect is a live one —- // wiping its count and letting that IP exceed the limit.- h.activeConnections.CompareAndDelete(clientIP, count)+func (h *WebSocketHandler) releaseRateLimitInternal(clientIP string) {+ h.activeConnectionsMu.Lock()+ defer h.activeConnectionsMu.Unlock()+ if count := h.activeConnections[clientIP]; count > 1 {+ h.activeConnections[clientIP] = count - 1+ } else {+ delete(h.activeConnections, clientIP)}}@@ -97,7 +93,7 @@ func (h *WebSocketHandler) acquireSystemStatsSamplerInternal(ctx context.ContextcloseReady := sync.OnceFunc(func() {close(ready)})- if !h.initializeCPUCacheCtx(samplerCtx) {+ if !h.initializeCPUCacheInternal(samplerCtx) {closeReady()return}@@ -362,8 +358,8 @@ func (h *WebSocketHandler) getGPUInfo(ctx context.Context) ([]systemtypes.GPUStareturn gpuData, len(gpuData)}-// initializeCPUCacheCtx performs initial CPU sampling and returns early if the sampler is canceled.-func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {+// initializeCPUCacheInternal permits unavailable CPU samples but stops on cancellation.+func (h *WebSocketHandler) initializeCPUCacheInternal(ctx context.Context) bool {result := make(chan float64, 1)go func() {@@ -377,10 +373,12 @@ func (h *WebSocketHandler) initializeCPUCacheCtx(ctx context.Context) bool {case <-ctx.Done():return falsecase val, ok := <-result:- if !ok || ctx.Err() != nil {+ if ctx.Err() != nil {return false}- h.cpuCache.Store(val)+ if ok {+ h.cpuCache.Store(val)+ }return true}}@@ -424,11 +422,10 @@ func (h *WebSocketHandler) getCachedCgroupLimitsInternal() *cgroup.Limits {func (h *WebSocketHandler) SystemStats(c *echo.Context) error {clientIP := c.RealIP()- count, allowed := h.checkRateLimitInternal(clientIP)- if !allowed {+ if !h.checkRateLimitInternal(clientIP) {return c.JSON(http.StatusTooManyRequests, map[string]any{"success": false, "error": "Too many concurrent stats connections from this IP"})}- defer h.releaseRateLimitInternal(clientIP, count)+ defer h.releaseRateLimitInternal(clientIP)conn, unregister, ok := h.acceptWSInternal(c, systemtypes.WSKindSystemStats, "")if !ok {diff --git a/backend/internal/auth/service.go b/backend/internal/auth/service.goindex 59b23a9ca..3216e4193 100644--- a/backend/internal/auth/service.go+++ b/backend/internal/auth/service.go@@ -10,6 +10,7 @@ import ("strconv""strings""sync"+ "sync/atomic""time""uuid"@@ -75,7 +76,9 @@ type AuthService struct {// skips signature verification and the user/session lookups; revocation in// this process purges entries, revocation by another process is visible// once the TTL lapses.- tokenCache *hot.HotCache[string, verifiedTokenEntry]+ tokenCache *hot.HotCache[string, verifiedTokenEntry]+ cacheGen atomic.Uint64+ cacheFillMu sync.Mutex}func NewAuthService(userService *user.UserService, settingsService *settings.SettingsService, eventService *event.EventService, sessionService *session.SessionService, roleService *role.RoleService, cfg *config.Config) *AuthService {@@ -820,6 +823,7 @@ func (s *AuthService) cachedVerificationInternal(tokenHash string) (*common.User}func (s *AuthService) verifyTokenClaimsInternal(ctx context.Context, tokenHash string, claims *accessTokenClaims) (*common.User, string, error) {+ gen := s.cacheGen.Load()if claims.AppVersion != "" && claims.AppVersion != config.Version {slog.InfoContext(ctx, "Token version mismatch detected", "tokenVersion", claims.AppVersion, "currentVersion", config.Version, "user", claims.Username)return nil, "", common.ErrTokenVersionMismatch@@ -847,7 +851,11 @@ func (s *AuthService) verifyTokenClaimsInternal(ctx context.Context, tokenHash sreturn nil, "", err}- s.tokenCache.Set(tokenHash, verifiedTokenEntry{User: *dbUser, SessionID: userSession.ID, TokenExpiresAt: claims.ExpiresAt, SessionExpiresAt: userSession.ExpiresAt})+ s.cacheFillMu.Lock()+ if s.cacheGen.Load() == gen {+ s.tokenCache.Set(tokenHash, verifiedTokenEntry{User: *dbUser, SessionID: userSession.ID, TokenExpiresAt: claims.ExpiresAt, SessionExpiresAt: userSession.ExpiresAt})+ }+ s.cacheFillMu.Unlock()return dbUser, userSession.ID, nil}@@ -895,9 +903,16 @@ func (s *AuthService) InvalidateUserTokenCache(userID string) {if s.tokenCache == nil || strings.TrimSpace(userID) == "" {return}+ s.invalidateTokenCacheInternal(func(entry verifiedTokenEntry) bool { return entry.User.ID == userID })+}++func (s *AuthService) invalidateTokenCacheInternal(matches func(verifiedTokenEntry) bool) {+ s.cacheFillMu.Lock()+ defer s.cacheFillMu.Unlock()+ s.cacheGen.Add(1)keys := make([]string, 0)s.tokenCache.Range(func(key string, entry verifiedTokenEntry) bool {- if entry.User.ID == userID {+ if matches(entry) {keys = append(keys, key)}return true@@ -909,15 +924,11 @@ func (s *AuthService) RevokeSession(ctx context.Context, sessionID string) errorif s.sessionService == nil {return nil}- keys := make([]string, 0)- s.tokenCache.Range(func(key string, entry verifiedTokenEntry) bool {- if entry.SessionID == sessionID {- keys = append(keys, key)- }- return true- })- s.tokenCache.DeleteMany(keys)- return s.sessionService.RevokeSession(ctx, sessionID)+ if err := s.sessionService.RevokeSession(ctx, sessionID); err != nil {+ return err+ }+ s.invalidateTokenCacheInternal(func(entry verifiedTokenEntry) bool { return entry.SessionID == sessionID })+ return nil}// LogoutAllOtherSessions revokes every active session for userID except@@ -926,15 +937,13 @@ func (s *AuthService) LogoutAllOtherSessions(ctx context.Context, userID, currenif s.sessionService == nil {return nil}- keys := make([]string, 0)- s.tokenCache.Range(func(key string, entry verifiedTokenEntry) bool {- if entry.User.ID == userID && entry.SessionID != currentSessionID {- keys = append(keys, key)- }- return true+ if err := s.sessionService.RevokeAllUserSessionsExcept(ctx, userID, currentSessionID); err != nil {+ return err+ }+ s.invalidateTokenCacheInternal(func(entry verifiedTokenEntry) bool {+ return entry.User.ID == userID && entry.SessionID != currentSessionID})- s.tokenCache.DeleteMany(keys)- return s.sessionService.RevokeAllUserSessionsExcept(ctx, userID, currentSessionID)+ return nil}func (s *AuthService) createSessionAndTokensInternal(ctx context.Context, user *common.User, meta auth.SessionMeta) (*TokenPair, error) {diff --git a/backend/internal/passkey/handler.go b/backend/internal/passkey/handler.goindex f5233b4d2..e6d5bec30 100644--- a/backend/internal/passkey/handler.go+++ b/backend/internal/passkey/handler.go@@ -633,6 +633,7 @@ func (h *PasskeyHandler) DisableMFA(ctx context.Context, input *MFASettingsInputif err := h.passkeyService.DisableMFA(ctx, userModel.ID, sessionID, input.StepUpToken); err != nil {return nil, passkeyHTTPErrorInternal(err)}+ h.authService.InvalidateUserTokenCache(userModel.ID)return &handlerutil.Out[base.MessageResponse]{Body: base.ApiResponse[base.MessageResponse]{Success: true, Data: base.MessageResponse{Message: "Passkey MFA disabled"}}}, nil}diff --git a/backend/internal/role/service.go b/backend/internal/role/service.goindex d16119863..e963ddfe5 100644--- a/backend/internal/role/service.go+++ b/backend/internal/role/service.go@@ -9,6 +9,8 @@ import ("maps""slices""strings"+ "sync"+ "sync/atomic""time""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -32,9 +34,12 @@ const ()type RoleService struct {- db *database.DB- userCache *hot.HotCache[string, *authz.PermissionSet]- apiKeyCache *hot.HotCache[string, *authz.PermissionSet]+ db *database.DB+ userCache *hot.HotCache[string, *authz.PermissionSet]+ apiKeyCache *hot.HotCache[string, *authz.PermissionSet]+ cacheFillMu sync.Mutex+ userCacheGen atomic.Uint64+ apiKeyCacheGen atomic.Uint64}func NewRoleService(db *database.DB) *RoleService {@@ -256,6 +261,7 @@ func (s *RoleService) UpdateRole(ctx context.Context, id, name string, descriptireturn nil, err}var out Role+ var affected []stringerr := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {var existing Roleif err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", id).First(&existing).Error; err != nil {@@ -276,9 +282,11 @@ func (s *RoleService) UpdateRole(ctx context.Context, id, name string, descriptireturn common.Classify(common.ErrRoleNameTaken, errors.New("Role name already in use"))}}- if _, err := lockAssignedUserRowsInternal(tx, id); err != nil {+ ids, err := lockAssignedUserRowsInternal(tx, id)+ if err != nil {return err}+ affected = idsexisting.Name = input.Nameexisting.Description = input.Descriptionexisting.Permissions = permissions@@ -291,7 +299,9 @@ func (s *RoleService) UpdateRole(ctx context.Context, id, name string, descriptiif err != nil {return nil, err}- s.invalidateUsersAssignedToInternal(ctx, id)+ for _, userID := range affected {+ s.InvalidateUser(userID)+ }return &out, nil}@@ -327,7 +337,7 @@ func (s *RoleService) DeleteRole(ctx context.Context, id string) error {// cache-miss cannot re-populate with stale data from the not-yet-visible// delete. Consistent with UpdateRole / SetUserAssignments.for _, uid := range affected {- s.userCache.Delete(uid)+ s.InvalidateUser(uid)}return nil}@@ -406,7 +416,7 @@ func (s *RoleService) replaceUserAssignmentsForSourceInternal(ctx context.Contexif err != nil {return err}- s.userCache.Delete(userID)+ s.InvalidateUser(userID)return nil}@@ -538,11 +548,16 @@ func (s *RoleService) ResolvePermissions(ctx context.Context, user *common.User)if ps, ok, _ := s.userCache.Get(user.ID); ok {return ps, nil}+ gen := s.userCacheGen.Load()ps, err := s.ResolveUserPermissionsInDB(ctx, s.db.WithContext(ctx), user.ID)if err != nil {return nil, err}- s.userCache.Set(user.ID, ps)+ s.cacheFillMu.Lock()+ if s.userCacheGen.Load() == gen {+ s.userCache.Set(user.ID, ps)+ }+ s.cacheFillMu.Unlock()return ps, nil}@@ -653,11 +668,16 @@ func (s *RoleService) ResolveApiKeyPermissions(ctx context.Context, apiKeyID strif ps, ok, _ := s.apiKeyCache.Get(apiKeyID); ok {return ps, nil}+ gen := s.apiKeyCacheGen.Load()permissions, err := s.resolveApiKeyPermissionsInDBInternal(s.db.WithContext(ctx), apiKeyID)if err != nil {return nil, err}- s.apiKeyCache.Set(apiKeyID, permissions)+ s.cacheFillMu.Lock()+ if s.apiKeyCacheGen.Load() == gen {+ s.apiKeyCache.Set(apiKeyID, permissions)+ }+ s.cacheFillMu.Unlock()return permissions, nil}@@ -687,7 +707,7 @@ func (s *RoleService) SetApiKeyPermissions(ctx context.Context, apiKeyID string,if err != nil {return err}- s.apiKeyCache.Delete(apiKeyID)+ s.InvalidateApiKey(apiKeyID)return nil}@@ -913,31 +933,20 @@ func (s *RoleService) ReconcileEnvOidcMappings(ctx context.Context, rawSpec stri// auth_service after a login that mutates assignments, and from any mutation// path that doesn't already invalidate explicitly.func (s *RoleService) InvalidateUser(userID string) {+ s.cacheFillMu.Lock()+ defer s.cacheFillMu.Unlock()+ s.userCacheGen.Add(1)s.userCache.Delete(userID)}// InvalidateApiKey drops the cached PermissionSet for one API key.func (s *RoleService) InvalidateApiKey(apiKeyID string) {+ s.cacheFillMu.Lock()+ defer s.cacheFillMu.Unlock()+ s.apiKeyCacheGen.Add(1)s.apiKeyCache.Delete(apiKeyID)}-// invalidateUsersAssignedToInternal invalidates every user holding an assignment to-// the given role. Called after a role's permissions change.-func (s *RoleService) invalidateUsersAssignedToInternal(ctx context.Context, roleID string) {- var userIDs []string- if err := s.db.WithContext(ctx).- Model(&UserRoleAssignment{}).- Distinct("user_id").- Where("role_id = ?", roleID).- Pluck("user_id", &userIDs).Error; err != nil {- slog.WarnContext(ctx, "failed to collect users for cache invalidation", "error", err, "role_id", roleID)- return- }- for _, id := range userIDs {- s.userCache.Delete(id)- }-}-// ---------- helpers ----------func validatePermissionsInternal(perms []string) error {diff --git a/backend/internal/template/service.go b/backend/internal/template/service.goindex 26c367e35..8f3591437 100644--- a/backend/internal/template/service.go+++ b/backend/internal/template/service.go@@ -13,6 +13,7 @@ import ("path/filepath""strings""sync"+ "sync/atomic""time""uuid"@@ -47,7 +48,8 @@ type TemplateService struct {lookupIP httputils.LookupIPFuncsettingsService *settings.SettingsService- remoteCache *hot.HotCache[struct{}, []ComposeTemplate]+ remoteCache *hot.HotCache[uint64, []ComposeTemplate]+ remoteGeneration atomic.Uint64registryMu sync.RWMutexregistryFetchMeta map[string]*registryFetchMeta@@ -65,7 +67,7 @@ const (remoteIDPrefix = "remote")-var errNoRemoteTemplates = errors.New("remote template registries returned no templates")+var errRemoteCacheInvalidatedInternal = errors.New("remote template cache invalidated")func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.Client, settingsService *settings.SettingsService) *TemplateService {if httpClient == nil {@@ -82,19 +84,20 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.C}service.safeHTTPClient = service.newSafeHTTPClientInternal()revalidationCtx := context.WithoutCancel(ctx)- loader := func(_ []struct{}) (map[struct{}][]ComposeTemplate, error) {+ loader := func(generations []uint64) (map[uint64][]ComposeTemplate, error) {loadCtx, cancel := context.WithTimeout(revalidationCtx, 2*time.Minute)defer cancel()- templates, err := service.loadRemoteTemplates(loadCtx)- if err != nil {- return nil, err- }- if len(templates) == 0 {- return nil, errNoRemoteTemplates+ catalogs := make(map[uint64][]ComposeTemplate, len(generations))+ for _, generation := range generations {+ templates, err := service.loadRemoteTemplatesInternal(loadCtx, generation)+ if err != nil {+ return nil, err+ }+ catalogs[generation] = templates}- return map[struct{}][]ComposeTemplate{{}: templates}, nil+ return catalogs, nil}- service.remoteCache = hot.NewHotCache[struct{}, []ComposeTemplate](hot.LRU, 1).+ service.remoteCache = hot.NewHotCache[uint64, []ComposeTemplate](hot.LRU, 1).WithTTL(remoteCacheDuration).WithLoaders(loader).WithRevalidation(24*time.Hour, loader).@@ -125,39 +128,47 @@ func (s *TemplateService) getTemplatesDirectoryInternal(ctx context.Context) (streturn projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSettingInternal(ctx)))}-func (s *TemplateService) ensureRemoteTemplatesLoaded(_ context.Context) error {+func (s *TemplateService) remoteTemplatesInternal(ctx context.Context, refresh bool) ([]ComposeTemplate, error) {if s.remoteCache == nil {- return errors.New("remote template cache is not initialized")+ return nil, errors.New("remote template cache is not initialized")}- templates, found, err := s.remoteCache.Get(struct{}{})- if err != nil {- return fmt.Errorf("failed to load remote templates: %w", err)+ for {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ generation := s.remoteGeneration.Load()+ var templates []ComposeTemplate+ var err error+ if refresh {+ templates, err = s.loadRemoteTemplatesInternal(ctx, generation)+ s.registryMu.Lock()+ if generation == s.remoteGeneration.Load() && err == nil {+ s.remoteCache.Set(generation, templates)+ }+ s.registryMu.Unlock()+ } else {+ var found bool+ templates, found, err = s.remoteCache.Get(generation)+ if err == nil && !found {+ err = errors.New("remote template catalog not found")+ }+ }+ if generation != s.remoteGeneration.Load() {+ continue+ }+ if err != nil {+ return nil, fmt.Errorf("failed to load remote templates: %w", err)+ }+ return templates, nil}- return kit.Ternary(!found || len(templates) == 0, errNoRemoteTemplates, nil)-}--func (s *TemplateService) refreshRemoteTemplates(ctx context.Context) error {- templates, err := s.loadRemoteTemplates(ctx)- if err != nil {- return fmt.Errorf("failed to load remote templates: %w", err)- }-- if len(templates) == 0 {- return errNoRemoteTemplates- }- if s.remoteCache == nil {- return errors.New("remote template cache is not initialized")- }- s.remoteCache.Set(struct{}{}, templates)- return nil}func (s *TemplateService) GetAllTemplates(ctx context.Context) ([]ComposeTemplate, error) {- return s.getMergedTemplates(ctx)+ return s.getMergedTemplatesInternal(ctx)}func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params pagination.QueryParams) ([]tmpl.Template, pagination.Response, error) {- templates, err := s.getMergedTemplates(ctx)+ templates, err := s.getMergedTemplatesInternal(ctx)if err != nil {return nil, pagination.Response{}, err}@@ -235,11 +246,12 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeTreturn nil, fmt.Errorf("failed to query local template: %w", err)}- if err := s.ensureRemoteTemplatesLoaded(ctx); err != nil && !errors.Is(err, errNoRemoteTemplates) {+ templates, err := s.remoteTemplatesInternal(ctx, false)+ if err != nil {return nil, fmt.Errorf("template %q lookup failed: registry refresh error: %w", id, err)}- if found := s.lookupRemoteFromCacheInternal(id); found != nil {+ if found := s.lookupRemoteTemplateInternal(templates, id); found != nil {return found, nil}@@ -247,27 +259,21 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*ComposeT// before we return "not found" — the cache may be stale or the previous refresh// silently returned empty.if strings.HasPrefix(id, remoteIDPrefix+":") {- slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateID", id, "cacheSize", s.remoteCacheSizeInternal())- if refreshErr := s.refreshRemoteTemplates(ctx); refreshErr != nil && !errors.Is(refreshErr, errNoRemoteTemplates) {+ slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateID", id, "cacheSize", len(templates))+ templates, refreshErr := s.remoteTemplatesInternal(ctx, true)+ if refreshErr != nil {return nil, fmt.Errorf("template %q not found and registry refresh failed: %w", id, refreshErr)}- if found := s.lookupRemoteFromCacheInternal(id); found != nil {+ if found := s.lookupRemoteTemplateInternal(templates, id); found != nil {return found, nil}- return nil, common.Classify(common.ErrTemplateNotFound, fmt.Errorf("Template not found: %w", fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, s.remoteCacheSizeInternal()))) //nolint:staticcheck // Preserve the existing error message.+ return nil, common.Classify(common.ErrTemplateNotFound, fmt.Errorf("Template not found: %w", fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, len(templates)))) //nolint:staticcheck // Preserve the existing error message.}return nil, common.Classify(common.ErrTemplateNotFound, errors.New("Template not found")) //nolint:staticcheck // Preserve the existing error message.}-func (s *TemplateService) lookupRemoteFromCacheInternal(id string) *ComposeTemplate {- if s.remoteCache == nil {- return nil- }- templates, found := s.remoteCache.Peek(struct{}{})- if !found {- return nil- }+func (s *TemplateService) lookupRemoteTemplateInternal(templates []ComposeTemplate, id string) *ComposeTemplate {for i := range templates {if templates[i].ID == id {cloned := cloneRemoteTemplates(templates[i : i+1])@@ -277,14 +283,6 @@ func (s *TemplateService) lookupRemoteFromCacheInternal(id string) *ComposeTemplreturn nil}-func (s *TemplateService) remoteCacheSizeInternal() int {- if s.remoteCache == nil {- return 0- }- templates, _ := s.remoteCache.Peek(struct{}{})- return len(templates)-}-func (s *TemplateService) CreateTemplate(ctx context.Context, template *ComposeTemplate) error {if err := normalization.Normalize(template); err != nil {return err@@ -491,7 +489,7 @@ func (s *TemplateService) CreateRegistry(ctx context.Context, registry *Templatereturn err}- s.invalidateRemoteCache()+ s.invalidateRemoteCacheInternal()return nil}@@ -526,7 +524,7 @@ func (s *TemplateService) UpdateRegistry(ctx context.Context, id string, updatesreturn err}- s.invalidateRemoteCache()+ s.invalidateRemoteCacheInternal()return nil}@@ -568,19 +566,21 @@ func (s *TemplateService) DeleteRegistry(ctx context.Context, id string) error {return err}- s.invalidateRemoteCache()+ s.invalidateRemoteCacheInternal()return nil}-func (s *TemplateService) loadRemoteTemplates(ctx context.Context) ([]ComposeTemplate, error) {+func (s *TemplateService) loadRemoteTemplatesInternal(ctx context.Context, generation uint64) ([]ComposeTemplate, error) {registries, err := s.GetRegistries(ctx)if err != nil {return nil, err}var (- mu sync.Mutex- templates []ComposeTemplate+ mu sync.Mutex+ templates []ComposeTemplate+ fetchErrors []error+ successfulRegistries int)g, groupCtx := errgroup.WithContext(ctx)@@ -594,21 +594,29 @@ func (s *TemplateService) loadRemoteTemplates(ctx context.Context) ([]ComposeTemg.Go(func() (workerErr error) {defer utils.RecoverToError(&workerErr, "template worker")- remoteTemplates, err := s.fetchRegistryTemplates(groupCtx, ®)+ remoteTemplates, err := s.fetchRegistryTemplatesInternal(groupCtx, ®, generation)if err != nil {slog.WarnContext(groupCtx, "failed to fetch templates from registry", "registry", reg.Name, "url", reg.URL, "error", err)s.registryMu.Lock()- s.registryErrors[reg.ID] = err.Error()+ if generation == s.remoteGeneration.Load() {+ s.registryErrors[reg.ID] = err.Error()+ }s.registryMu.Unlock()+ mu.Lock()+ fetchErrors = append(fetchErrors, fmt.Errorf("registry %q: %w", reg.Name, err))+ mu.Unlock()return nil // Don't fail the whole group if one registry fails}s.registryMu.Lock()- delete(s.registryErrors, reg.ID)+ if generation == s.remoteGeneration.Load() {+ delete(s.registryErrors, reg.ID)+ }s.registryMu.Unlock()mu.Lock()defer mu.Unlock()+ successfulRegistries++for _, template := range remoteTemplates {template.Registry = cloneRegistry(®)template.RegistryID = mo.EmptyableToOption(strings.TrimSpace(reg.ID)).ToPointer()@@ -621,6 +629,12 @@ func (s *TemplateService) loadRemoteTemplates(ctx context.Context) ([]ComposeTemif err := g.Wait(); err != nil {return nil, err}+ if generation != s.remoteGeneration.Load() {+ return nil, errRemoteCacheInvalidatedInternal+ }+ if successfulRegistries == 0 && len(fetchErrors) > 0 {+ return nil, errors.Join(fetchErrors...)+ }return templates, nil}@@ -651,10 +665,14 @@ func (s *TemplateService) doGET(ctx context.Context, url string) ([]byte, error)return body, nil}-// fetchRegistryTemplates performs a conditional GET using If-Modified-Since.+// fetchRegistryTemplatesInternal performs a conditional GET using If-Modified-Since.// If the server replies 304 Not Modified, cached templates for the registry are reused.-func (s *TemplateService) fetchRegistryTemplates(ctx context.Context, reg *TemplateRegistry) ([]ComposeTemplate, error) {+func (s *TemplateService) fetchRegistryTemplatesInternal(ctx context.Context, reg *TemplateRegistry, generation uint64) ([]ComposeTemplate, error) {s.registryMu.RLock()+ if generation != s.remoteGeneration.Load() {+ s.registryMu.RUnlock()+ return nil, errRemoteCacheInvalidatedInternal+ }fetchMeta := s.registryFetchMeta[reg.ID]s.registryMu.RUnlock()@@ -704,6 +722,10 @@ func (s *TemplateService) fetchRegistryTemplates(ctx context.Context, reg *TemplTemplates: cloneRemoteTemplates(templates),}s.registryMu.Lock()+ if generation != s.remoteGeneration.Load() {+ s.registryMu.Unlock()+ return nil, errRemoteCacheInvalidatedInternal+ }s.registryFetchMeta[reg.ID] = newMetas.registryMu.Unlock()@@ -1001,14 +1023,15 @@ func cloneRegistry(registry *TemplateRegistry) *TemplateRegistry {return new(*registry)}-func (s *TemplateService) invalidateRemoteCache() {+func (s *TemplateService) invalidateRemoteCacheInternal() {+ s.registryMu.Lock()+ defer s.registryMu.Unlock()+ s.remoteGeneration.Add(1)+ s.registryFetchMeta = make(map[string]*registryFetchMeta)+ s.registryErrors = make(map[string]string)if s.remoteCache != nil {s.remoteCache.Purge()}-- s.registryMu.Lock()- s.registryFetchMeta = make(map[string]*registryFetchMeta)- s.registryMu.Unlock()}func (s *TemplateService) SyncLocalTemplatesFromFilesystem(ctx context.Context) error {@@ -1180,7 +1203,7 @@ func (s *TemplateService) GetTemplateContentWithParsedData(ctx context.Context,}, nil}-func (s *TemplateService) getMergedTemplates(ctx context.Context) ([]ComposeTemplate, error) {+func (s *TemplateService) getMergedTemplatesInternal(ctx context.Context) ([]ComposeTemplate, error) {if err := s.syncFilesystemTemplatesInternal(ctx); err != nil {slog.WarnContext(ctx, "failed to sync filesystem templates", "error", err)}@@ -1191,10 +1214,10 @@ func (s *TemplateService) getMergedTemplates(ctx context.Context) ([]ComposeTempreturn nil, fmt.Errorf("failed to get local templates: %w", err)}- if err := s.ensureRemoteTemplatesLoaded(ctx); err != nil {+ remoteTemplates, err := s.remoteTemplatesInternal(ctx, false)+ if err != nil {slog.WarnContext(ctx, "failed to load remote templates", "error", err)} else {- remoteTemplates, _ := s.remoteCache.Peek(struct{}{})copied := cloneRemoteTemplates(remoteTemplates)if len(copied) > 0 {diff --git a/backend/internal/template/service_test.go b/backend/internal/template/service_test.goindex 20a60bc65..69f805cb0 100644--- a/backend/internal/template/service_test.go+++ b/backend/internal/template/service_test.go@@ -151,7 +151,7 @@ services:Enabled: true,}- templates, err := service.fetchRegistryTemplates(context.Background(), registry)+ templates, err := service.fetchRegistryTemplatesInternal(context.Background(), registry, service.remoteGeneration.Load())require.NoError(t, err)require.Len(t, templates, 2)require.NotNil(t, templates[0].Metadata)@@ -160,7 +160,7 @@ services:require.Nil(t, templates[1].Metadata.IconURL)require.EqualValues(t, 1, composeHits.Load())- cachedTemplates, err := service.fetchRegistryTemplates(context.Background(), registry)+ cachedTemplates, err := service.fetchRegistryTemplatesInternal(context.Background(), registry, service.remoteGeneration.Load())require.NoError(t, err)require.Len(t, cachedTemplates, 2)require.NotNil(t, cachedTemplates[0].Metadata)@@ -259,7 +259,7 @@ func TestGetAllTemplatesPaginated_FiltersByType(t *testing.T) {require.NoError(t, db.WithContext(context.Background()).Create(&localTemplates).Error)service := NewTemplateService(context.Background(), db, http.DefaultClient, nil)- service.remoteCache.Set(struct{}{}, []ComposeTemplate{+ service.remoteCache.Set(service.remoteGeneration.Load(), []ComposeTemplate{{ID: "remote-one", CreatedAt: now, UpdatedAt: &now,Name: "Remote One",diff --git a/backend/internal/volume/helper_container.go b/backend/internal/volume/helper_container.goindex c14d1012e..84dc80ea7 100644--- a/backend/internal/volume/helper_container.go+++ b/backend/internal/volume/helper_container.go@@ -12,6 +12,7 @@ import ("strings""time"+ cerrdefs "github.com/containerd/errdefs""github.com/getarcaneapp/arcane/backend/v2/internal/common"dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"@@ -175,7 +176,11 @@ func (s *VolumeService) acquireVolumeHelperInternal(ctx context.Context, volumeN// creation itself can pull an image, so it must not run under helperMu.func (s *VolumeService) resolveHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (string, error) {resultCh := s.helperGroup.DoChan(volumeName, func() (any, error) {- if containerID, ok := s.getReusableHelperInternal(ctx, dockerClient, volumeName).Get(); ok {+ helper, err := s.getReusableHelperInternal(ctx, dockerClient, volumeName)+ if err != nil {+ return nil, err+ }+ if containerID, ok := helper.Get(); ok {return containerID, nil}@@ -274,8 +279,13 @@ func (s *VolumeService) startHelperContainerInternal(ctx context.Context, docker}if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- return "", nil, fmt.Errorf("failed to start temp container: %w", err)+ startErr := fmt.Errorf("failed to start temp container: %w", err)+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)+ defer cancel()+ if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !cerrdefs.IsNotFound(cleanupErr) {+ return "", nil, errors.Join(startErr, fmt.Errorf("failed to remove temp container after start failure: %w", cleanupErr))+ }+ return "", nil, startErr}cleanup := func() {@@ -285,25 +295,38 @@ func (s *VolumeService) startHelperContainerInternal(ctx context.Context, dockerreturn resp.ID, cleanup, nil}-func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) mo.Option[string] {+func (s *VolumeService) getReusableHelperInternal(ctx context.Context, dockerClient *client.Client, volumeName string) (mo.Option[string], error) {s.helperMu.Lock()helper := s.helperByVolume[volumeName]s.helperMu.Unlock()if helper == nil || helper.id == "" {- return mo.None[string]()+ return mo.None[string](), nil}inspect, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, helper.id, client.ContainerInspectOptions{})+ if err != nil && !cerrdefs.IsNotFound(err) {+ return mo.None[string](), fmt.Errorf("failed to inspect volume helper container: %w", err)+ }+ if err == nil && (inspect.Container.State == nil || !inspect.Container.State.Running) {+ removeCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultDockerAPI)+ _, removeErr := dockerClient.ContainerRemove(removeCtx, helper.id, volumehelper.RemoveOptions())+ cancel()+ if removeErr != nil && !cerrdefs.IsNotFound(removeErr) {+ return mo.None[string](), fmt.Errorf("failed to remove stopped volume helper container: %w", removeErr)+ }+ }if err != nil || inspect.Container.State == nil || !inspect.Container.State.Running {s.helperMu.Lock()- delete(s.helperByVolume, volumeName)+ if s.helperByVolume[volumeName] == helper {+ delete(s.helperByVolume, volumeName)+ }s.helperMu.Unlock()- return mo.None[string]()+ return mo.None[string](), nil}s.touchHelperInternal(volumeName)- return mo.Some(helper.id)+ return mo.Some(helper.id), nil}// touchHelperInternal records that the helper for volumeName just serviced adiff --git a/backend/internal/volume/workspace.go b/backend/internal/volume/workspace.goindex 1f75d01b5..9ff636e2a 100644--- a/backend/internal/volume/workspace.go+++ b/backend/internal/volume/workspace.go@@ -25,6 +25,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/event"dockerutil "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper"acfsutils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/acfs"workspacepkg "github.com/getarcaneapp/arcane/backend/v2/pkg/workspace"@@ -83,11 +84,16 @@ func (s *VolumeService) readVolumeWorkspaceFromContainerInternal(ctx context.ConmaxEntries = 10000}- pipeReader, pipeWriter := io.Pipe()dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {return nil, err}+ workCtx, cancel := context.WithCancel(ctx)+ defer cancel()+ pipeReader, pipeWriter := io.Pipe()+ // io.PipeReader.CloseWithError always returns nil.+ stopClose := context.AfterFunc(workCtx, func() { _ = pipeReader.CloseWithError(workCtx.Err()) })+ defer stopClose()var stderr bytes.Bufferdone := make(chan error, 1)go func() {@@ -95,7 +101,7 @@ func (s *VolumeService) readVolumeWorkspaceFromContainerInternal(ctx context.Con"acfs", "walk", "--root", "/volume", "--path", "/","--max-depth", strconv.Itoa(maxDepth), "--max-entries", strconv.Itoa(maxEntries),}- exitCode, execErr := dockerutil.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{+ exitCode, execErr := dockerutil.ExecInContainer(workCtx, dockerClient, containerID, client.ExecCreateOptions{AttachStdout: true,AttachStderr: true,Cmd: cmd,@@ -109,11 +115,12 @@ func (s *VolumeService) readVolumeWorkspaceFromContainerInternal(ctx context.Conworkspace, parseErr := decodeVolumeWorkspaceWalkInternal(pipeReader, maxEntries, s.volumeWorkspaceMaxFileSizeBytesInternal())if parseErr != nil {+ cancel()_ = pipeReader.CloseWithError(parseErr)}execErr := <-done_ = pipeReader.Close()- if execErr != nil {+ if execErr != nil && (parseErr == nil || ctx.Err() != nil || !errors.Is(execErr, context.Canceled)) {return nil, classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace")}if parseErr != nil {@@ -314,6 +321,9 @@ func (s *VolumeService) DownloadVolumeWorkspaceFile(ctx context.Context, volumeN}type volumeWorkspaceReadStreamInternal struct {+ ctx context.Context+ cancel context.CancelFunc+ stopClose func() boolpipe *io.PipeReaderdone <-chan errorcleanup func()@@ -351,20 +361,28 @@ func (r *volumeWorkspaceReadStreamInternal) Read(buffer []byte) (int, error) {func (r *volumeWorkspaceReadStreamInternal) Close() error {r.once.Do(func() {+ r.stopClose()+ r.cancel()_ = r.pipe.Close()r.err = <-r.done+ if r.ctx.Err() == nil && errors.Is(r.err, context.Canceled) {+ r.err = nil+ }r.cleanup()})return r.err}func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, containerID, relativePath string, limit int64, cleanup func()) (io.ReadCloser, int64, error) {- pipeReader, pipeWriter := io.Pipe()dockerClient, err := s.dockerService.GetClient(ctx)if err != nil {cleanup()return nil, 0, err}+ workCtx, cancel := context.WithCancel(ctx)+ pipeReader, pipeWriter := io.Pipe()+ // io.PipeReader.CloseWithError always returns nil.+ stopClose := context.AfterFunc(workCtx, func() { _ = pipeReader.CloseWithError(workCtx.Err()) })done := make(chan error, 1)cmd := []string{"acfs", "read", "--root", "/volume", "--path", "/" + relativePath}if limit > 0 {@@ -372,7 +390,7 @@ func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, co}go func() {var stderr bytes.Buffer- exitCode, execErr := dockerutil.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{+ exitCode, execErr := dockerutil.ExecInContainer(workCtx, dockerClient, containerID, client.ExecCreateOptions{AttachStdout: true,AttachStderr: true,Cmd: cmd,@@ -383,21 +401,24 @@ func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, coif execErr != nil {execErr = classifyVolumeWorkspaceExecErrorInternal(execErr, stderr.String(), "read volume workspace file")}+ stopClose()_ = pipeWriter.CloseWithError(execErr)done <- execErr}()payloadSize, err := acfs.ReadStreamHeader(pipeReader)if err != nil {+ cancel()_ = pipeReader.CloseWithError(err)execErr := <-donecleanup()- if execErr != nil {+ if execErr != nil && (ctx.Err() != nil || !errors.Is(execErr, context.Canceled)) {return nil, 0, execErr}return nil, 0, fmt.Errorf("parse volume workspace read header: %w", err)}if payloadSize > uint64(1<<63-1) {+ cancel()_ = pipeReader.Close()<-donecleanup()@@ -405,6 +426,9 @@ func (s *VolumeService) startVolumeWorkspaceReadInternal(ctx context.Context, co}size := int64(payloadSize)return &volumeWorkspaceReadStreamInternal{+ ctx: ctx,+ cancel: cancel,+ stopClose: stopClose,pipe: pipeReader,done: done,cleanup: cleanup,@@ -1049,8 +1073,13 @@ func (s *VolumeService) createVolumeWorkspaceMutationContainerInternal(ctx contereturn "", nil, fmt.Errorf("create volume workspace helper: %w", err)}if _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {- _, _ = dockerClient.ContainerRemove(ctx, resp.ID, volumehelper.RemoveOptions())- return "", nil, fmt.Errorf("start volume workspace helper: %w", err)+ startErr := fmt.Errorf("start volume workspace helper: %w", err)+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDockerAPI)+ defer cancel()+ if _, cleanupErr := dockerClient.ContainerRemove(cleanupCtx, resp.ID, volumehelper.RemoveOptions()); cleanupErr != nil && !cerrdefs.IsNotFound(cleanupErr) {+ return "", nil, errors.Join(startErr, fmt.Errorf("remove volume workspace helper after start failure: %w", cleanupErr))+ }+ return "", nil, startErr}return resp.ID, func() {_, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), resp.ID, volumehelper.RemoveOptions())diff --git a/backend/pkg/dockerutil/exec.go b/backend/pkg/dockerutil/exec.goindex 817da5d7a..988290b0f 100644--- a/backend/pkg/dockerutil/exec.go+++ b/backend/pkg/dockerutil/exec.go@@ -63,10 +63,16 @@ func ExecInContainer(ctx context.Context, dockerClient *client.Client, containerreturn 0, fmt.Errorf("failed to attach to exec: %w", err)}defer attachResp.Close()+ stopClose := context.AfterFunc(ctx, attachResp.Close)+ defer stopClose()- if err := WaitStdCopy(StartStdCopy(attachResp.Reader, stdout, stderr)); err != nil {+ copyErr := WaitStdCopy(StartStdCopy(attachResp.Reader, stdout, stderr))+ if err := ctx.Err(); err != nil {return 0, fmt.Errorf("failed to read exec output: %w", err)}+ if copyErr != nil {+ return 0, fmt.Errorf("failed to read exec output: %w", copyErr)+ }inspect, err := dockerClient.ExecInspect(ctx, execResp.ID, client.ExecInspectOptions{})if err != nil {diff --git a/backend/pkg/fswatch/watcher.go b/backend/pkg/fswatch/watcher.goindex dee21bdbd..c29858acb 100644--- a/backend/pkg/fswatch/watcher.go+++ b/backend/pkg/fswatch/watcher.go@@ -30,6 +30,7 @@ type Watcher struct {debounce time.DurationstopCh chan struct{}stoppedCh chan struct{}+ cancel context.CancelFuncwatchAliases map[string]stringpendingPaths map[string]struct{}mu sync.Mutex@@ -105,7 +106,9 @@ func (fw *Watcher) Start(ctx context.Context) error {"error", err)}- go fw.watchLoop(ctx)+ watchCtx, cancel := context.WithCancel(ctx)+ fw.cancel = cancel+ go fw.watchLoopInternal(watchCtx)fw.started = trueslog.InfoContext(ctx, "Filesystem watcher started", "path", fw.watchedPath)@@ -125,9 +128,11 @@ func (fw *Watcher) StopWatching() error {fw.mu.Lock()fw.stopped = truestarted := fw.started+ cancel := fw.cancelfw.mu.Unlock()if started {+ cancel()close(fw.stopCh)<-fw.stoppedCh // Wait for watchLoop to finish}@@ -145,7 +150,7 @@ func (fw *Watcher) StopWatching() error {return fw.stopErr}-func (fw *Watcher) watchLoop(ctx context.Context) {+func (fw *Watcher) watchLoopInternal(ctx context.Context) {defer close(fw.stoppedCh)debounceTimer := time.NewTimer(fw.debounce)@@ -221,18 +226,31 @@ func (fw *Watcher) fireDebounceInternal(ctx context.Context, debouncePending *bo"goroutines", runtime.NumGoroutine())*lastGoroutineLog = time.Now()}- if fw.onChange != nil {- fw.callbacks.Go(func() { fw.onChange(ctx) })- }+ var paths []stringif fw.onChangePaths != nil {- paths := fw.drainPendingPathsInternal()- if len(paths) > 0 {- fw.callbacks.Go(func() { fw.onChangePaths(ctx, paths) })- }+ paths = fw.drainPendingPathsInternal()}+ fw.notifyChangesInternal(ctx, paths)return false}+func (fw *Watcher) notifyChangesInternal(ctx context.Context, paths []string) {+ if fw.onChange != nil {+ fw.callbacks.Go(func() {+ if ctx.Err() == nil {+ fw.onChange(ctx)+ }+ })+ }+ if fw.onChangePaths != nil && len(paths) > 0 {+ fw.callbacks.Go(func() {+ if ctx.Err() == nil {+ fw.onChangePaths(ctx, paths)+ }+ })+ }+}+func (fw *Watcher) recordPendingPathInternal(path string) {if path == "" {return@@ -486,12 +504,7 @@ func (fw *Watcher) reconnectInternal(ctx context.Context) bool {slog.WarnContext(ctx, "Some filesystem subscriptions could not be restored", "error", err)}// Changes while disconnected need a full reconciliation.- if fw.onChange != nil {- fw.callbacks.Go(func() { fw.onChange(ctx) })- }- if fw.onChangePaths != nil {- fw.callbacks.Go(func() { fw.onChangePaths(ctx, []string{fw.watchedPath}) })- }+ fw.notifyChangesInternal(ctx, []string{fw.watchedPath})return true}if closeErr := watcher.Close(); closeErr != nil {diff --git a/backend/pkg/libarcane/rustic/run.go b/backend/pkg/libarcane/rustic/run.goindex 24e0c61ab..a405e9d66 100644--- a/backend/pkg/libarcane/rustic/run.go+++ b/backend/pkg/libarcane/rustic/run.go@@ -5,8 +5,12 @@ import ("cmp""context""fmt"+ "log/slog""strings"+ "time"+ cerrdefs "github.com/containerd/errdefs"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/volumehelper""github.com/moby/moby/api/pkg/stdcopy""github.com/moby/moby/api/types/container"@@ -49,7 +53,19 @@ func Run(ctx context.Context, dockerClient *client.Client, password string, commreturn "", fmt.Errorf("failed to create Rustic container: %w", err)}defer func() {- _, _ = dockerClient.ContainerRemove(context.WithoutCancel(ctx), created.ID, volumehelper.RemoveOptions())+ // Keep caller locks and stopped consumers protected until removal is confirmed.+ // Request cancellation cannot prove that Rustic has stopped writing.+ cleanupCtx := context.WithoutCancel(ctx)+ for {+ attemptCtx, cancel := context.WithTimeout(cleanupCtx, timeouts.DefaultDockerAPI)+ _, err := dockerClient.ContainerRemove(attemptCtx, created.ID, volumehelper.RemoveOptions())+ cancel()+ if err == nil || cerrdefs.IsNotFound(err) {+ return+ }+ slog.WarnContext(cleanupCtx, "failed to remove Rustic container, retrying", "container_id", created.ID, "error", err)+ time.Sleep(5 * time.Second)+ }}()if _, err := dockerClient.ContainerStart(ctx, created.ID, client.ContainerStartOptions{}); err != nil {return "", fmt.Errorf("failed to start Rustic container: %w", err)diff --git a/go.work.sum b/go.work.sumindex 0d92bbde4..5833d5349 100644--- a/go.work.sum+++ b/go.work.sum@@ -383,6 +383,11 @@ cyphar.com/go-pathrs v0.2.4/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGcyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik=cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc=dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY=+emperror.dev/emperror v0.33.0 h1:urYop6KLYxKVpZbt9ADC4eVG3WDnJFE6Ye3j07wUu/I=+emperror.dev/emperror v0.33.0/go.mod h1:CeOIKPcppTE8wn+3xBNcdzdHMMIP77sLOHS0Ik56m+w=+emperror.dev/errors v0.8.0/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE=+emperror.dev/errors v0.8.1 h1:UavXZ5cSX/4u9iyvH6aDcuGkVjeexUGJ7Ij7G4VfQT0=+emperror.dev/errors v0.8.1/go.mod h1:YcRvLPh626Ubn2xqtoprejnA5nFha+TJ+2vew48kWuE=filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=@@ -2763,8 +2768,6 @@ go.getarcane.app/kit v0.1.0 h1:82oO/i1fS4xbzAG4Xc8ALOWOwW4kwQYDdBpVHE19QkI=go.getarcane.app/kit v0.1.0/go.mod h1:oiH4hLowxfQYFtmldyk/y79XYBHYfzgd86MrSQS9UEg=go.getarcane.app/kit v0.2.0 h1:3PPHo5nHuns1vjqmCMFPFdnmbH/qQOserONCba/gBi4=go.getarcane.app/kit v0.2.0/go.mod h1:TGqQCbAJs3ldzkFncXZ93wTVSDhAMQlqyNdVO2CNjIk=-go.getarcane.app/kit v0.3.2 h1:L7CAvGawDg70K/vVxDeM2xZBCHqwuthc9B5OwQwL0Hc=-go.getarcane.app/kit v0.3.2/go.mod h1:KTjbXTEeC3i9e1QxZowDJWjFyABlD6UF3ISVtoIZs10=go.getarcane.app/streams v0.4.0 h1:MtBX1SVgt3nrcRNr5SYAvckGp9a5qeSjMTw/LAGwlzk=go.getarcane.app/streams v0.4.0/go.mod h1:LHMKblQvsL1nqKh/Y+KgQbwFSMI2I7bs9u8ZwZqzjnQ=go.getarcane.app/streams v0.4.2 h1:+xrRHiOt25cLbFTDpPLzv7eFkk7biwDz5YjBXkbWUPw=@@ -2936,6 +2939,7 @@ go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWpgo.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=@@ -2943,6 +2947,7 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=