Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

2.6 KB
Raw Blame
package dockeragent

import (
	"context"
	"strings"
	"time"

	containertypes "github.com/moby/moby/api/types/container"
)

// cleanupOrphanedBackups searches for and removes any Pulse backup containers
// (created during updates) that are older than 15 minutes.
func (a *Agent) cleanupOrphanedBackups(ctx context.Context) {
	if a.cfg.HelperInventory != nil {
		a.logger.Debug().Msg("Skipping orphaned backup cleanup in monitoring-only collector profile")
		return
	}
	if !a.tryStartCleanupTask() {
		a.logger.Debug().Msg("Skipping orphaned backup cleanup - previous cleanup still running")
		return
	}
	defer a.finishCleanupTask()
	a.cleanupOrphanedBackupsTask(ctx)
}

func (a *Agent) cleanupOrphanedBackupsTask(ctx context.Context) {
	a.logger.Debug().Msg("Checking for orphaned backup containers")

	// List all containers (including stopped ones)
	list, err := dockerCallWithRetry(ctx, dockerCleanupCallTimeout, func(callCtx context.Context) ([]containertypes.Summary, error) {
		return a.docker.ContainerList(callCtx, dockerContainerListOptions{
			All: true,
		})
	})
	if err != nil {
		a.logger.Warn().Err(annotateDockerConnectionError(err)).Msg("Failed to list containers for cleanup")
		return
	}

	for _, c := range list {
		if !isBackupContainer(c.Names) {
			continue
		}

		// Check age based on the timestamp in the name, not the container's creation date
		// (Renaming a container does not change its creation date)
		parts := strings.Split(c.Names[0], backupContainerMarker)
		if len(parts) < 2 {
			continue
		}

		timestampStr := parts[len(parts)-1]
		// Backup names are currently stamped with the agent host's local wall
		// clock, so cleanup must parse in the local location to avoid treating
		// valid stale backups as fresh on non-UTC systems.
		backupTime, err := time.ParseInLocation("20060102_150405", timestampStr, time.Local)
		if err != nil {
			// If we can't parse the timestamp, fall back to creation date as a safety
			created := time.Unix(c.Created, 0)
			if time.Since(created) < 15*time.Minute {
				continue
			}
			backupTime = created
		}

		if time.Since(backupTime) > 15*time.Minute {
			a.logger.Info().
				Str("container", c.Names[0]).
				Time("backupTime", backupTime).
				Msg("removing orphaned backup container")

			_, err := dockerCallWithRetry(ctx, dockerCleanupCallTimeout, func(callCtx context.Context) (struct{}, error) {
				err := a.docker.ContainerRemove(callCtx, c.ID, dockerContainerRemoveOptions{Force: true})
				return struct{}{}, err
			})
			if err != nil {
				a.logger.Warn().Err(annotateDockerConnectionError(err)).Str("id", c.ID).Msg("Failed to remove orphaned backup container")
			}
		}
	}
}